From 65048ada20c2e014da706d0ade988454bdc43f09 Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 09:59:50 -0400 Subject: [PATCH 01/15] Add asynchronous whole-site accessibility audits --- .github/workflows/ci.yml | 3 + deploy/report-bucket-lifecycle.json | 8 + entry_points/api_server.py | 196 +++++++- entry_points/generate_report.py | 25 +- entry_points/run_pipeline.py | 23 +- index.html | 131 +++++- processing_scripts/llm_client/client.py | 37 +- pyproject.toml | 5 +- requirements.txt | 64 ++- tests/__init__.py | 1 + tests/test_site_audit.py | 79 ++++ uv.lock | 257 +++++++++++ vision_aid/site_audit/__init__.py | 16 + vision_aid/site_audit/crawler.py | 450 +++++++++++++++++++ vision_aid/site_audit/jobs.py | 567 ++++++++++++++++++++++++ vision_aid/site_audit/report.py | 228 ++++++++++ 16 files changed, 2057 insertions(+), 33 deletions(-) create mode 100644 deploy/report-bucket-lifecycle.json create mode 100644 tests/__init__.py create mode 100644 tests/test_site_audit.py create mode 100644 vision_aid/site_audit/__init__.py create mode 100644 vision_aid/site_audit/crawler.py create mode 100644 vision_aid/site_audit/jobs.py create mode 100644 vision_aid/site_audit/report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 774f495..cd920c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,9 @@ jobs: print('all entry points import cleanly') " + - name: Unit tests + run: uv run python -m unittest discover -v + # Exercises extractors, slicers, prompt templates and the programmatic # checkers end to end. --dry-run makes no API calls, so this needs no key # and costs nothing. diff --git a/deploy/report-bucket-lifecycle.json b/deploy/report-bucket-lifecycle.json new file mode 100644 index 0000000..c6ccc16 --- /dev/null +++ b/deploy/report-bucket-lifecycle.json @@ -0,0 +1,8 @@ +{ + "rule": [ + { + "action": {"type": "Delete"}, + "condition": {"age": 30} + } + ] +} diff --git a/entry_points/api_server.py b/entry_points/api_server.py index 0f61b4f..0855f98 100644 --- a/entry_points/api_server.py +++ b/entry_points/api_server.py @@ -19,6 +19,8 @@ """ import json +import hashlib +import hmac import os import re import shutil @@ -26,6 +28,7 @@ import tempfile from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from urllib.parse import urlparse from dotenv import load_dotenv @@ -37,6 +40,8 @@ from entry_points.generate_report import generate_report # noqa: E402 from vision_aid.ingestion.file_crawler import fetch_page, fetch_pages_nested # noqa: E402 from processing_scripts.llm_client.client import is_openai_model, is_gemini_model # noqa: E402 +from vision_aid.site_audit.crawler import validate_public_url # noqa: E402 +from vision_aid.site_audit.jobs import get_coordinator # noqa: E402 # ── Multi-page splitting ───────────────────────────────────────────────────── @@ -243,10 +248,22 @@ def do_OPTIONS(self): # noqa: N802 def do_GET(self): # noqa: N802 path = self.path.split("?")[0] + if path not in ("/api/health", "/styles.css") and not self._site_access_allowed(): + if path in ("/", "/index.html") or path.endswith("/report"): + self._serve_login_page() + else: + self._send_json({"success": False, "error": "Authentication required"}, 401) + return if path in ("/", "/index.html"): self._serve_file(STATIC_DIR / "index.html", "text/html; charset=utf-8") elif path == "/styles.css": self._serve_file(STATIC_DIR / "styles.css", "text/css; charset=utf-8") + elif path == "/api/health": + self._handle_health() + elif re.fullmatch(r"/api/site-audits/[A-Za-z0-9_-]+/report", path): + self._handle_site_audit_report(path) + elif re.fullmatch(r"/api/site-audits/[A-Za-z0-9_-]+", path): + self._handle_site_audit_status(path) else: self.send_error(404, "Not Found") @@ -265,7 +282,17 @@ def _serve_file(self, file_path: Path, content_type: str): # POST ───────────────────────────────────────────────────────────────────── def do_POST(self): # noqa: N802 - if self.path == "/api/audit": + if self.path == "/api/login": + self._handle_login() + elif self.path.startswith("/api/internal/site-audits/"): + operation = self.path.rsplit("/", 1)[-1] + if operation not in {"discover", "page", "finalize"}: + self.send_error(404, "Not Found") + return + self._handle_internal_site_audit(operation) + elif not self._site_access_allowed(): + self._send_json({"success": False, "error": "Authentication required"}, 401) + elif self.path == "/api/audit": self._handle_audit() elif self.path == "/api/audit/url": self._handle_url_audit(nested=False) @@ -273,9 +300,70 @@ def do_POST(self): # noqa: N802 self._handle_url_audit(nested=True) elif self.path == "/api/validate-key": self._handle_validate_key() + elif self.path == "/api/site-audits": + self._handle_create_site_audit() else: self.send_error(404, "Not Found") + def _site_access_cookie(self) -> str: + password = os.getenv("DAT_SITE_PASSWORD", "").strip() + if not password: + return "" + return hashlib.sha256(f"vision-aid-dat:{password}".encode("utf-8")).hexdigest() + + def _site_access_allowed(self) -> bool: + """Return whether the shared-password cookie is valid.""" + expected = self._site_access_cookie() + if not expected: + return True + cookies = self.headers.get("Cookie", "") + supplied = "" + for item in cookies.split(";"): + name, _, value = item.strip().partition("=") + if name == "dat_access": + supplied = value + break + return bool(supplied and hmac.compare_digest(expected, supplied)) + + def _serve_login_page(self): + body = b""" +Vision Aid DAT sign in + +

Vision Aid Digital Accessibility Testing

Team testing access

+
+
+""" + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _handle_login(self): + try: + data = self._read_json_body() + except ValueError as exc: + self._send_json({"success": False, "error": str(exc)}, 400) + return + expected_password = os.getenv("DAT_SITE_PASSWORD", "").strip() + supplied = str(data.get("password", "")) + if not expected_password or not hmac.compare_digest(expected_password, supplied): + self._send_json({"success": False, "error": "Incorrect password"}, 401) + return + body = json.dumps({"success": True}).encode("utf-8") + secure_flag = "; Secure" if os.getenv("K_SERVICE") or self.headers.get( + "X-Forwarded-Proto", "" + ).lower() == "https" else "" + self.send_response(200) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header( + "Set-Cookie", + f"dat_access={self._site_access_cookie()}; Path=/; Max-Age=28800; HttpOnly; SameSite=Lax{secure_flag}", + ) + self.end_headers() + self.wfile.write(body) + def _start_ndjson_stream(self): """Send NDJSON response headers and return a send_event callable.""" self.send_response(200) @@ -340,6 +428,11 @@ def _handle_url_audit(self, nested: bool): if not url: self._send_json({"success": False, "error": "url is required"}, 400) return + try: + url = validate_public_url(url) + except ValueError as exc: + self._send_json({"success": False, "error": str(exc)}, 400) + return model = data.get("model", "claude-haiku-4-5-20251001") api_key = _resolve_api_key(data, model) @@ -570,6 +663,107 @@ def _handle_validate_key(self): except Exception as exc: self._send_json({"valid": False, "error": str(exc)}) + def _read_json_body(self) -> dict: + """Read and parse one bounded JSON request body.""" + length = int(self.headers.get("Content-Length", 0)) + if length <= 0 or length > 64 * 1024: + raise ValueError("A JSON request body is required") + try: + value = json.loads(self.rfile.read(length)) + except json.JSONDecodeError as exc: + raise ValueError("Invalid JSON body") from exc + if not isinstance(value, dict): + raise ValueError("JSON body must be an object") + return value + + def _handle_health(self): + """Return non-secret service health and async feature readiness.""" + coordinator = get_coordinator() + self._send_json( + { + "status": "ok", + "service": "vision-aid-dat", + "async_site_audit_configured": coordinator.configured, + "async_model": coordinator.model, + "max_site_pages": 200, + } + ) + + def _handle_create_site_audit(self): + """Create a durable whole-site audit and return immediately.""" + try: + data = self._read_json_body() + job = get_coordinator().create_job( + base_url=data.get("url", ""), + email=data.get("email", ""), + ) + except ValueError as exc: + self._send_json({"success": False, "error": str(exc)}, 400) + return + except Exception as exc: + print(f" Site audit creation failed: {exc}") + self._send_json( + {"success": False, "error": "Could not queue the site audit"}, 503 + ) + return + self._send_json({"success": True, "job": job}, 202) + + def _handle_site_audit_status(self, path: str): + job_id = path.rstrip("/").rsplit("/", 1)[-1] + coordinator = get_coordinator() + job = coordinator.get_job(job_id) + if not job: + self._send_json({"success": False, "error": "Job not found"}, 404) + return + self._send_json({"success": True, "job": coordinator.public_job(job)}) + + def _handle_site_audit_report(self, path: str): + job_id = path.split("/")[-2] + report = get_coordinator().report_bytes(job_id) + if report is None: + self._send_json({"success": False, "error": "Report is not ready"}, 404) + return + self.send_response(200) + self.send_header("Content-Type", "application/zip") + self.send_header( + "Content-Disposition", 'attachment; filename="DAT-whole-site-report.zip"' + ) + self.send_header("Content-Length", str(len(report))) + self._cors_headers() + self.end_headers() + self.wfile.write(report) + + def _handle_internal_site_audit(self, operation: str): + """Handle authenticated callbacks from the dedicated Cloud Tasks queue.""" + coordinator = get_coordinator() + supplied_token = self.headers.get("X-DAT-Job-Token", "") + if not coordinator.internal_token_valid(supplied_token): + self._send_json({"success": False, "error": "Not found"}, 404) + return + try: + data = self._read_json_body() + job_id = data.get("job_id", "") + if operation == "discover": + result = coordinator.run_discovery(job_id) + elif operation == "page": + retry_count = int(self.headers.get("X-CloudTasks-TaskRetryCount", "0")) + result = coordinator.run_page( + job_id=job_id, + page_id=data.get("page_id", ""), + retry_count=retry_count, + audit_callable=run_audit, + ) + else: + result = coordinator.finalize(job_id) + except ValueError as exc: + self._send_json({"success": False, "error": str(exc)}, 400) + return + except Exception as exc: + print(f" Internal site audit {operation} failed: {exc}") + self._send_json({"success": False, "error": "Task failed; retrying"}, 500) + return + self._send_json({"success": True, "result": result}) + def _send_json(self, obj: dict, status: int = 200): body = json.dumps(obj, indent=2, ensure_ascii=False).encode("utf-8") self.send_response(status) diff --git a/entry_points/generate_report.py b/entry_points/generate_report.py index fb86401..ddf097d 100644 --- a/entry_points/generate_report.py +++ b/entry_points/generate_report.py @@ -871,13 +871,22 @@ def deduplicate_with_llm( if is_openai: import openai client = openai.OpenAI(api_key=api_key) - response = client.chat.completions.create( - model=model, - max_tokens=512, - temperature=0.0, - messages=[{"role": "user", "content": prompt}], - ) - response_text = response.choices[0].message.content or "[]" + if model.startswith("gpt-5"): + response = client.responses.create( + model=model, + input=prompt, + max_output_tokens=512, + reasoning={"effort": "low"}, + ) + response_text = response.output_text or "[]" + else: + response = client.chat.completions.create( + model=model, + max_tokens=512, + temperature=0.0, + messages=[{"role": "user", "content": prompt}], + ) + response_text = response.choices[0].message.content or "[]" else: import anthropic from processing_scripts.llm_client.client import supports_temperature @@ -1086,4 +1095,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/entry_points/run_pipeline.py b/entry_points/run_pipeline.py index 1334021..e74ee75 100644 --- a/entry_points/run_pipeline.py +++ b/entry_points/run_pipeline.py @@ -66,6 +66,8 @@ "gpt-4.1": (2.00, 8.00), "gpt-4.1-mini": (0.40, 1.60), "gpt-4.1-nano": (0.10, 0.40), + "gpt-5.6-sol": (4.00, 20.00), + "gpt-5.6": (4.00, 20.00), "o3": (2.00, 8.00), "o3-mini": (1.10, 4.40), "o4-mini": (1.10, 4.40), @@ -181,7 +183,26 @@ def _call_anthropic(self, prompt: str, start: float) -> dict: } def _call_openai(self, prompt: str, start: float) -> dict: - """Call the OpenAI Chat Completions API.""" + """Call OpenAI, using Responses for GPT-5 reasoning models.""" + if self.model.startswith("gpt-5"): + response = self._client.responses.create( + model=self.model, + input=prompt, + max_output_tokens=self.max_tokens, + reasoning={"effort": "low"}, + ) + return { + "success": True, + "response": response.output_text, + "model": response.model, + "usage": { + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + }, + "stop_reason": response.status, + "duration_seconds": round(time.time() - start, 2), + } + response = self._client.chat.completions.create( model=self.model, max_tokens=self.max_tokens, diff --git a/index.html b/index.html index e5cdb93..5fad1bc 100644 --- a/index.html +++ b/index.html @@ -327,6 +327,10 @@

Accessibility Audit Tool

URL — With Crawl + @@ -465,7 +469,23 @@

Accessibility Audit Tool

-
+ + +

- Without a key, only programmatic checks run. Never stored or - logged. + Without a key, only programmatic checks run. Keys are never + logged or returned by the server.

Accessibility Audit Tool const modelSelect = document.getElementById("modelSelect"); const apiKeyLabel = document.getElementById("apiKeyLabel"); const apiKeyInput = document.getElementById("apiKeyInput"); + const apiKeyHint = document.getElementById("apiKeyHint"); + let siteAuditConfig = { + model: "gpt-5.6-sol", + api_key_configured: false, + api_key_verified: false, + api_key_masked: "", + }; + + function showSavedKeyStatus() { + if (getAuditMode() !== "site" || apiKeyInput.value.trim()) return; + const provider = getProvider(modelSelect.value); + apiKeyValidateStatus.hidden = false; + if (provider !== "openai") { + apiKeyValidateStatus.className = "key-validate-status validating"; + apiKeyValidateStatus.textContent = + "Enter and validate an API key for this provider."; + } else if (siteAuditConfig.api_key_verified) { + apiKeyValidateStatus.className = "key-validate-status valid"; + apiKeyValidateStatus.textContent = + "✓ Saved OpenAI API key verified for " + modelSelect.value; + } else if (siteAuditConfig.api_key_configured) { + apiKeyValidateStatus.className = "key-validate-status invalid"; + apiKeyValidateStatus.textContent = + "Saved key configured but not verified for this model. Select Validate to retry."; + } else { + apiKeyValidateStatus.className = "key-validate-status invalid"; + apiKeyValidateStatus.textContent = + "No saved OpenAI API key is available."; + } + } + + async function loadSiteAuditConfig(model, refresh) { + const params = new URLSearchParams(); + if (model) params.set("model", model); + if (refresh) params.set("refresh", "1"); + try { + const response = await fetch( + "/api/site-audit-config" + (params.toString() ? "?" + params : "") + ); + const result = await response.json(); + if (response.ok && result.success) { + siteAuditConfig = result.config; + updateApiKeyField(); + showSavedKeyStatus(); + } + } catch { + if (getAuditMode() === "site") { + apiKeyValidateStatus.hidden = false; + apiKeyValidateStatus.className = "key-validate-status invalid"; + apiKeyValidateStatus.textContent = + "Could not check the saved API key status."; + } + } + } function updateApiKeyField() { const provider = getProvider(modelSelect.value); + const isSite = getAuditMode() === "site"; if (provider === "openai") { apiKeyLabel.textContent = "OpenAI API Key"; - apiKeyInput.placeholder = "sk-…"; + apiKeyInput.placeholder = + isSite && siteAuditConfig.api_key_configured + ? (siteAuditConfig.api_key_masked || "••••••••••••••••") + " (saved)" + : "sk-…"; } else if (provider === "gemini") { apiKeyLabel.textContent = "Gemini API Key"; apiKeyInput.placeholder = "Gemini API key…"; @@ -706,9 +764,17 @@

Accessibility Audit Tool

apiKeyLabel.textContent = "Anthropic API Key"; apiKeyInput.placeholder = "sk-ant-api03-…"; } + apiKeyHint.textContent = isSite + ? "The saved server key is used when this field is empty. Enter a different key only to override it for this audit; the saved key is never sent to your browser." + : "Without a key, only programmatic checks run. Keys are never logged or returned by the server."; } - modelSelect.addEventListener("change", updateApiKeyField); + modelSelect.addEventListener("change", () => { + updateApiKeyField(); + if (getAuditMode() === "site" && !apiKeyInput.value.trim()) { + loadSiteAuditConfig(modelSelect.value, false); + } + }); // ── API key validation ──────────────────────────────────────────────────── const validateKeyBtn = document.getElementById("validateKeyBtn"); @@ -718,15 +784,20 @@

Accessibility Audit Tool

validateKeyBtn.addEventListener("click", async () => { const key = apiKeyInput.value.trim(); - if (!key) { + const model = modelSelect.value; + const provider = getProvider(model); + const useSaved = + !key && + getAuditMode() === "site" && + provider === "openai" && + siteAuditConfig.api_key_configured; + if (!key && !useSaved) { apiKeyValidateStatus.hidden = false; apiKeyValidateStatus.className = "key-validate-status invalid"; apiKeyValidateStatus.textContent = "Please enter an API key first."; return; } - const provider = getProvider(modelSelect.value); - validateKeyBtn.disabled = true; apiKeyValidateStatus.hidden = false; apiKeyValidateStatus.className = "key-validate-status validating"; @@ -736,7 +807,12 @@

Accessibility Audit Tool

const resp = await fetch("/api/validate-key", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ api_key: key, provider }), + body: JSON.stringify({ + api_key: key, + provider, + model, + use_saved: useSaved, + }), }); const result = await resp.json(); if (result.valid) { @@ -776,6 +852,7 @@

Accessibility Audit Tool

apiKeyValidateStatus.hidden = true; apiKeyValidateStatus.textContent = ""; apiKeyValidateStatus.className = "key-validate-status"; + if (!apiKeyInput.value.trim()) showSavedKeyStatus(); }); // Input mode switching @@ -792,7 +869,16 @@

Accessibility Audit Tool

htmlInputSection.hidden = !isHtml; urlInputSection.hidden = isHtml; siteEmailSection.hidden = !isSite; - modelKeySection.hidden = isSite; + modelKeySection.hidden = false; + if (isSite) { + modelSelect.value = siteAuditConfig.model || "gpt-5.6-sol"; + apiKeyInput.value = ""; + updateApiKeyField(); + loadSiteAuditConfig(modelSelect.value, false); + } else { + updateApiKeyField(); + apiKeyValidateStatus.hidden = true; + } document.getElementById("auditSubmitBtn").textContent = isSite ? "Queue Whole-Site Audit" : "Run Audit"; @@ -806,6 +892,8 @@

Accessibility Audit Tool

return document.querySelector('input[name="auditMode"]:checked').value; } + loadSiteAuditConfig(); + // ── Drag-and-drop / file input ──────────────────────────────────────────── let htmlFileContent = null; @@ -960,7 +1048,8 @@

Accessibility Audit Tool

return; } endpoint = "/api/site-audits"; - requestBody = { url: urlVal, email }; + requestBody = { url: urlVal, email, model }; + if (keyValue) requestBody.api_key = keyValue; } else { endpoint = mode === "url-nested" diff --git a/pyproject.toml b/pyproject.toml index fbd9a7d..e734905 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.11" dependencies = [ "anthropic~=0.84.0", "beautifulsoup4>=4.9.2", + "cryptography>=45.0.0", "lxml>=4.9.0", "openai>=1.0.0", "google-genai>=1.0.0", diff --git a/requirements.txt b/requirements.txt index 4f12e1e..5314bf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ # This file was autogenerated by uv via the following command: -# uv export --format requirements.txt --no-dev --no-emit-project --no-hashes -o requirements.txt +# uv export --frozen --format requirements-txt --no-hashes --output-file requirements.txt +-e . annotated-types==0.8.0 # via pydantic anthropic==0.84.0 @@ -24,7 +25,9 @@ charset-normalizer==3.4.9 colorama==0.4.6 ; sys_platform == 'win32' # via tqdm cryptography==50.0.0 - # via google-auth + # via + # google-auth + # visionaid distro==1.9.0 # via # anthropic diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index a218935..4fa2d28 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -3,12 +3,24 @@ from __future__ import annotations import io +import json +import os import unittest import zipfile from unittest import mock -from vision_aid.site_audit.crawler import canonicalize_url, validate_public_url -from vision_aid.site_audit.jobs import validate_request_email +from cryptography.fernet import Fernet + +from vision_aid.site_audit.crawler import ( + PageCandidate, + _ai_order_candidates, + _discover_wordpress_urls, + _is_candidate_url, + _sitemap_locations, + canonicalize_url, + validate_public_url, +) +from vision_aid.site_audit.jobs import SiteAuditCoordinator, validate_request_email from vision_aid.site_audit.report import build_site_report @@ -33,6 +45,80 @@ def test_public_addresses_are_accepted(self, getaddrinfo): "https://www.example.com/path", ) + def test_common_sitemap_indexes_are_tried_when_robots_is_blocked(self): + with mock.patch( + "vision_aid.site_audit.crawler._fetch_text_or_xml", + side_effect=RuntimeError("blocked"), + ): + locations = _sitemap_locations( + "https://example.com/", mock.Mock() + ) + self.assertIn("https://example.com/sitemap.xml", locations) + self.assertIn("https://example.com/sitemap_index.xml", locations) + self.assertIn("https://example.com/wp-sitemap.xml", locations) + + def test_wordpress_rest_fallback_discovers_public_pages(self): + def fetch(_session, url, **_kwargs): + if url.endswith("/wp-json/wp/v2/types"): + return json.dumps( + {"page": {"rest_base": "pages", "viewable": True}} + ), url + if "/wp-json/wp/v2/pages?" in url: + return json.dumps( + [ + {"link": "https://example.com/about/", "status": "publish"}, + {"link": "https://example.com/contact/", "status": "publish"}, + ] + ), url + raise AssertionError(url) + + with mock.patch( + "vision_aid.site_audit.crawler._fetch_text_or_xml", side_effect=fetch + ): + urls = _discover_wordpress_urls("https://example.com/", mock.Mock()) + self.assertEqual( + urls, + ["https://example.com/about/", "https://example.com/contact/"], + ) + + def test_admin_and_login_actions_are_not_page_candidates(self): + self.assertFalse( + _is_candidate_url( + "https://example.com/wp-admin/", "https://example.com/" + ) + ) + self.assertFalse( + _is_candidate_url( + "https://example.com/wp-login.php?action=lostpassword", + "https://example.com/", + ) + ) + self.assertFalse( + _is_candidate_url( + "https://example.com/?elementor_snippet=header", + "https://example.com/", + ) + ) + + def test_ai_order_cannot_drop_discovered_pages(self): + candidates = [ + PageCandidate("https://example.com/"), + PageCandidate("https://example.com/about/"), + PageCandidate("https://example.com/contact/"), + ] + client = mock.Mock() + client.responses.create.return_value = mock.Mock(output_text='{"order":[1]}') + with mock.patch("openai.OpenAI", return_value=client): + ordered = _ai_order_candidates( + candidates, + api_key="test-credential", + model="gpt-5.6-sol", + max_pages=200, + ) + self.assertEqual(len(ordered), 3) + self.assertEqual(ordered[0].url, "https://example.com/about/") + self.assertEqual({item.url for item in ordered}, {item.url for item in candidates}) + class EmailTests(unittest.TestCase): def test_allowed_domain(self): @@ -46,6 +132,50 @@ def test_disallowed_domain(self): validate_request_email("person@example.com", {"visionaid.org"}) +class CredentialTests(unittest.TestCase): + def test_override_is_encrypted_and_decrypted_only_for_the_job(self): + encryption_key = Fernet.generate_key().decode("ascii") + with mock.patch.dict( + os.environ, + { + "DAT_OPENAI_API_KEY": "saved-test-credential", + "DAT_CREDENTIAL_ENCRYPTION_KEY": encryption_key, + }, + ): + coordinator = SiteAuditCoordinator() + ciphertext = coordinator._encrypt_credential("override-test-credential") + self.assertNotIn("override-test-credential", ciphertext) + self.assertEqual( + coordinator._job_api_key( + { + "model": "gpt-5.6-sol", + "credential_override": ciphertext, + } + ), + "override-test-credential", + ) + + def test_public_config_masks_and_never_returns_saved_key(self): + encryption_key = Fernet.generate_key().decode("ascii") + with mock.patch.dict( + os.environ, + { + "DAT_MODEL": "gpt-5.6-sol", + "DAT_OPENAI_API_KEY": "saved-test-credential", + "DAT_CREDENTIAL_ENCRYPTION_KEY": encryption_key, + }, + ): + coordinator = SiteAuditCoordinator() + with mock.patch( + "vision_aid.site_audit.jobs.verify_model_key", + return_value=(True, "Verified"), + ): + config = coordinator.public_config(refresh=True) + self.assertTrue(config["api_key_verified"]) + self.assertEqual(config["api_key_masked"], "••••••••••••••••") + self.assertNotIn("saved-test-credential", repr(config)) + + class ReportTests(unittest.TestCase): def test_report_zip_contains_cover_csv_and_summary(self): csv_text = ( diff --git a/uv.lock b/uv.lock index 5cd62bc..736db6f 100644 --- a/uv.lock +++ b/uv.lock @@ -1114,6 +1114,7 @@ source = { editable = "." } dependencies = [ { name = "anthropic" }, { name = "beautifulsoup4" }, + { name = "cryptography" }, { name = "google-cloud-firestore" }, { name = "google-cloud-storage" }, { name = "google-cloud-tasks" }, @@ -1128,6 +1129,7 @@ dependencies = [ requires-dist = [ { name = "anthropic", specifier = "~=0.84.0" }, { name = "beautifulsoup4", specifier = ">=4.9.2" }, + { name = "cryptography", specifier = ">=45.0.0" }, { name = "google-cloud-firestore", specifier = ">=2.21.0" }, { name = "google-cloud-storage", specifier = ">=3.3.0" }, { name = "google-cloud-tasks", specifier = ">=2.19.0" }, diff --git a/vision_aid/site_audit/crawler.py b/vision_aid/site_audit/crawler.py index 5e5a3a7..9c722a9 100644 --- a/vision_aid/site_audit/crawler.py +++ b/vision_aid/site_audit/crawler.py @@ -1,4 +1,4 @@ -"""Safe, sitemap-aware whole-site discovery with AI-assisted selection.""" +"""Safe, multi-source whole-site discovery with AI-assisted ordering.""" from __future__ import annotations @@ -16,7 +16,17 @@ from bs4 import BeautifulSoup -USER_AGENT = "VisionAid-DAT/1.0 (+https://www.visionaid.org/)" +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/140.0.0.0 Safari/537.36" +) +REQUEST_HEADERS = { + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml,application/xml,text/xml;q=0.9,*/*;q=0.1", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", +} TRACKING_PARAMS = { "fbclid", "gclid", @@ -66,10 +76,41 @@ ".xml", ".zip", } -SITEMAP_LIMIT = 50 +SITEMAP_LIMIT = 100 CANDIDATE_LIMIT = 2_000 DISCOVERY_FETCH_LIMIT = 400 MAX_HTML_BYTES = 8 * 1024 * 1024 +COMMON_SITEMAP_PATHS = ( + "/sitemap.xml", + "/sitemap_index.xml", + "/sitemap-index.xml", + "/wp-sitemap.xml", +) +SKIP_PATH_PREFIXES = ( + "/wp-admin", + "/wp-json", + "/wp-login.php", + "/admin", + "/login", + "/logout", +) +SKIP_QUERY_ACTIONS = { + "delete", + "edit", + "lostpassword", + "logout", + "register", + "resetpass", +} +SKIP_QUERY_KEYS = { + "add-to-cart", + "customize_changeset_uuid", + "elementor-preview", + "elementor_snippet", + "preview", + "replytocom", + "wc-ajax", +} @dataclass(frozen=True) @@ -163,7 +204,7 @@ def _request( for _ in range(6): response = session.get( current, - headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml,application/xml,text/xml;q=0.9,*/*;q=0.1"}, + headers=REQUEST_HEADERS, timeout=timeout, allow_redirects=False, stream=True, @@ -234,10 +275,15 @@ def _fetch_text_or_xml( def _sitemap_locations(base_url: str, session: requests.Session) -> list[str]: - """Return robots-declared sitemaps plus the conventional sitemap URL.""" + """Return robots-declared and common sitemap locations. + + Some hosts block ``robots.txt`` or redirect ``sitemap.xml`` differently for + cloud data-center traffic. Trying the standard index variants directly + prevents one failed discovery endpoint from collapsing a crawl to one page. + """ parsed = urlparse(base_url) origin = f"{parsed.scheme}://{parsed.netloc}" - locations = [urljoin(origin, "/sitemap.xml")] + locations = [urljoin(origin, path) for path in COMMON_SITEMAP_PATHS] try: robots, _ = _fetch_text_or_xml(session, urljoin(origin, "/robots.txt")) for line in robots.splitlines(): @@ -292,7 +338,76 @@ def _is_candidate_url(url: str, base_url: str) -> bool: if not _is_same_site(url, base_url): return False path = parsed.path.lower() - return not any(path.endswith(extension) for extension in SKIP_EXTENSIONS) + normalized_path = path.rstrip("/") or "/" + if any(path.endswith(extension) for extension in SKIP_EXTENSIONS): + return False + if any(normalized_path == prefix or normalized_path.startswith(f"{prefix}/") for prefix in SKIP_PATH_PREFIXES): + return False + if normalized_path.endswith("/feed"): + return False + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + if any(key.lower() in SKIP_QUERY_KEYS for key in query): + return False + if str(query.get("action", "")).lower() in SKIP_QUERY_ACTIONS: + return False + return True + + +def _discover_wordpress_urls(base_url: str, session: requests.Session) -> list[str]: + """Discover public WordPress content through its read-only REST index. + + This is a fallback for WordPress sites whose HTML or sitemap requests are + blocked for cloud crawlers. Only canonical public ``link`` values returned + by the site's own API are accepted, and the normal same-site/safety filters + still apply. + """ + parsed = urlparse(base_url) + origin = f"{parsed.scheme}://{parsed.netloc}" + rest_bases = ["pages", "posts"] + try: + types_text, _ = _fetch_text_or_xml( + session, urljoin(origin, "/wp-json/wp/v2/types") + ) + types = json.loads(types_text) + if isinstance(types, dict): + discovered_bases = [ + str(item.get("rest_base", "")).strip("/") + for item in types.values() + if isinstance(item, dict) and item.get("viewable") is not False + ] + rest_bases = list( + dict.fromkeys(item for item in discovered_bases if item) + ) or rest_bases + except Exception: + pass + + page_urls: list[str] = [] + for rest_base in rest_bases[:25]: + if not re.fullmatch(r"[A-Za-z0-9_-]+", rest_base): + continue + for page_number in range(1, 21): + endpoint = urljoin( + origin, + f"/wp-json/wp/v2/{rest_base}?per_page=100&page={page_number}&_fields=link,status", + ) + try: + body, _ = _fetch_text_or_xml(session, endpoint) + rows = json.loads(body) + except Exception: + break + if not isinstance(rows, list): + break + for row in rows: + if not isinstance(row, dict) or row.get("status") not in (None, "publish"): + continue + candidate = canonicalize_url(str(row.get("link", ""))) + if _is_candidate_url(candidate, base_url) and candidate not in page_urls: + page_urls.append(candidate) + if len(page_urls) >= CANDIDATE_LIMIT: + return page_urls + if len(rows) < 100: + break + return page_urls def _extract_links(html: str, page_url: str, base_url: str) -> tuple[str, list[str]]: @@ -315,14 +430,14 @@ def _parse_json_object(text: str) -> dict: return json.loads(cleaned) -def _ai_filter_candidates( +def _ai_order_candidates( candidates: list[PageCandidate], *, api_key: str, model: str, max_pages: int, ) -> list[PageCandidate]: - """Use OpenAI to identify and order public, user-facing HTML pages.""" + """Use OpenAI to order candidates without ever dropping one.""" if not api_key or not candidates: return candidates[:max_pages] @@ -333,13 +448,12 @@ def _ai_filter_candidates( f"{index}\t{candidate.url}\t{candidate.title or '(title unknown)'}\t{candidate.source}" for index, candidate in enumerate(candidates) ) - prompt = f"""You are selecting pages for a whole-site WCAG accessibility audit. + prompt = f"""You are ordering pages for a whole-site WCAG accessibility audit. The deterministic crawler found the numbered same-site URL candidates below. -Return public, user-facing HTML pages only. Exclude logout/action endpoints, -admin-only paths, duplicate query variants, feeds, downloads, and obvious -non-pages. Prefer canonical, representative pages and put the most important -pages first. Never invent a URL. Return JSON only as -{{"include":[integer indexes]}} with at most {max_pages} unique indexes. +The crawler has already removed administrative actions, feeds, downloads, and +obvious non-pages. Put the most important public pages first, but return EVERY +candidate index exactly once. Never invent an index or URL. Return JSON only as +{{"order":[integer indexes]}}. Candidates: {rows} @@ -352,17 +466,14 @@ def _ai_filter_candidates( ) parsed = _parse_json_object(response.output_text) indexes: list[int] = [] - for value in parsed.get("include", []): + for value in parsed.get("order", []): try: index = int(value) except (TypeError, ValueError): continue if 0 <= index < len(candidates) and index not in indexes: indexes.append(index) - if len(indexes) >= max_pages: - break - if not indexes: - return candidates[:max_pages] + indexes.extend(index for index in range(len(candidates)) if index not in indexes) return [candidates[index] for index in indexes] @@ -374,12 +485,13 @@ def discover_site_urls( max_pages: int = 200, session: requests.Session | None = None, ) -> DiscoveryResult: - """Discover, AI-filter, and order up to ``max_pages`` site pages. + """Discover and order up to ``max_pages`` site pages. - Discovery combines the site's sitemap/robots declarations with a bounded - breadth-first traversal of same-site links. The model may filter and order - only URLs the deterministic crawler actually found; hallucinated URLs are - therefore impossible to add to the audit. + Discovery combines robots declarations, common recursive sitemap indexes, + public WordPress REST metadata when available, and a bounded breadth-first + traversal of same-site links. The model may reorder only URLs the + deterministic crawler actually found. It cannot remove pages or add + hallucinated URLs. """ if not 1 <= max_pages <= 200: raise ValueError("max_pages must be between 1 and 200") @@ -387,7 +499,8 @@ def discover_site_urls( active_session = session or requests.Session() sitemap_urls, sitemap_count = _read_sitemaps(normalized_base, active_session) - queue = deque([normalized_base, *sitemap_urls]) + wordpress_urls = _discover_wordpress_urls(normalized_base, active_session) + queue = deque([normalized_base, *sitemap_urls, *wordpress_urls]) candidates: dict[str, PageCandidate] = { normalized_base: PageCandidate(normalized_base, source="base") } @@ -395,6 +508,10 @@ def discover_site_urls( candidates.setdefault( sitemap_url, PageCandidate(sitemap_url, source="sitemap") ) + for wordpress_url in wordpress_urls: + candidates.setdefault( + wordpress_url, PageCandidate(wordpress_url, source="cms") + ) fetched: set[str] = set() while queue and len(fetched) < DISCOVERY_FETCH_LIMIT and len(candidates) < CANDIDATE_LIMIT: @@ -425,17 +542,21 @@ def discover_site_urls( ordered = list(candidates.values()) base_candidate = candidates[normalized_base] ordered = [base_candidate, *[item for item in ordered if item.url != normalized_base]] - ai_used = bool(api_key) + ai_used = bool(api_key and model.startswith(("gpt-", "o1", "o3", "o4"))) try: - selected = _ai_filter_candidates( - ordered, - api_key=api_key, - model=model, - max_pages=max_pages, + selected = ( + _ai_order_candidates( + ordered, + api_key=api_key, + model=model, + max_pages=max_pages, + ) + if ai_used + else ordered ) except Exception as exc: - print(f" WARNING: AI page selection failed ({exc}); using crawler order") - selected = ordered[:max_pages] + print(f" WARNING: AI page ordering failed ({exc}); using crawler order") + selected = ordered ai_used = False if normalized_base not in {item.url for item in selected}: diff --git a/vision_aid/site_audit/jobs.py b/vision_aid/site_audit/jobs.py index b6272d0..ad8f779 100644 --- a/vision_aid/site_audit/jobs.py +++ b/vision_aid/site_audit/jobs.py @@ -11,11 +11,14 @@ import re import secrets import smtplib +import urllib.error +import urllib.request from datetime import datetime, timedelta, timezone from email.message import EmailMessage from typing import Callable -from urllib.parse import urlparse +from urllib.parse import quote, urlparse +from cryptography.fernet import Fernet, InvalidToken from google.api_core.exceptions import AlreadyExists from google.cloud import firestore, storage, tasks_v2 from google.cloud.firestore_v1.base_query import FieldFilter @@ -28,6 +31,7 @@ ACTIVE_STATUSES = {"queued", "discovering", "auditing", "finalizing"} TERMINAL_PAGE_STATUSES = {"complete", "failed"} EMAIL_PATTERN = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$") +MODEL_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{1,99}$") def _now() -> datetime: @@ -50,6 +54,62 @@ def validate_request_email(address: str, allowed_domains: set[str]) -> str: return normalized +def _model_provider(model: str) -> str: + if model.startswith(("gpt-", "o1", "o3", "o4")): + return "openai" + if model.startswith("gemini-"): + return "gemini" + if model.startswith("claude-"): + return "anthropic" + raise ValueError("Select a supported OpenAI, Anthropic, or Gemini model") + + +def validate_model_name(model: str) -> str: + normalized = str(model or "").strip() + if not MODEL_PATTERN.fullmatch(normalized): + raise ValueError("Select a valid model") + _model_provider(normalized) + return normalized + + +def verify_model_key(api_key: str, model: str) -> tuple[bool, str]: + """Verify a provider credential without exposing it or generating tokens.""" + key = str(api_key or "").strip() + if not key: + return False, "No API key is configured" + try: + provider = _model_provider(validate_model_name(model)) + if provider == "openai": + request = urllib.request.Request( + f"https://api.openai.com/v1/models/{quote(model, safe='')}", + headers={"Authorization": f"Bearer {key}"}, + ) + elif provider == "gemini": + request = urllib.request.Request( + f"https://generativelanguage.googleapis.com/v1beta/models?key={quote(key, safe='')}", + ) + else: + request = urllib.request.Request( + "https://api.anthropic.com/v1/models", + headers={ + "x-api-key": key, + "anthropic-version": "2023-06-01", + }, + ) + with urllib.request.urlopen(request, timeout=15): + return True, "Verified" + except urllib.error.HTTPError as exc: + if exc.code in (401, 403): + return False, "Invalid or unauthorized API key" + if exc.code == 404 and provider == "openai": + return False, "The selected model is not available to this API key" + return False, f"Provider returned HTTP {exc.code}" + except ValueError as exc: + return False, str(exc) + except Exception: + return False, "Could not verify the API key with the provider" + + def _safe_task_id(value: str) -> str: return re.sub(r"[^A-Za-z0-9_-]", "-", value)[:500] @@ -143,6 +203,9 @@ def __init__(self) -> None: self.job_token = os.getenv("DAT_JOB_TOKEN", "").strip() self.model = os.getenv("DAT_MODEL", "gpt-5.6-sol") self.api_key = os.getenv("DAT_OPENAI_API_KEY", "").strip() + self.credential_encryption_key = os.getenv( + "DAT_CREDENTIAL_ENCRYPTION_KEY", "" + ).strip() self.allowed_domains = { item.strip().lower() for item in os.getenv("DAT_ALLOWED_EMAIL_DOMAINS", "").split(",") @@ -152,6 +215,8 @@ def __init__(self) -> None: self._db = None self._storage = None self._dispatcher = None + self._credential_cipher = None + self._key_verification_cache: dict[str, tuple[datetime, bool, str]] = {} @property def configured(self) -> bool: @@ -161,8 +226,81 @@ def configured(self) -> bool: and self.bucket_name and self.job_token and self.api_key + and self.credential_encryption_key ) + @property + def credential_cipher(self) -> Fernet: + if self._credential_cipher is None: + if not self.credential_encryption_key: + raise RuntimeError("Credential override encryption is not configured") + try: + self._credential_cipher = Fernet( + self.credential_encryption_key.encode("ascii") + ) + except (ValueError, TypeError) as exc: + raise RuntimeError("Credential override encryption is invalid") from exc + return self._credential_cipher + + def _verify_saved_key(self, model: str, *, refresh: bool = False) -> tuple[bool, str]: + selected_model = validate_model_name(model) + if _model_provider(selected_model) != "openai": + return False, "The saved credential is an OpenAI API key" + cached = self._key_verification_cache.get(selected_model) + if cached and not refresh and _now() - cached[0] < timedelta(minutes=5): + return cached[1], cached[2] + valid, message = verify_model_key(self.api_key, selected_model) + self._key_verification_cache[selected_model] = (_now(), valid, message) + return valid, message + + def public_config(self, *, model: str | None = None, refresh: bool = False) -> dict: + """Return non-secret model and saved-key state for the signed-in UI.""" + selected_model = validate_model_name(model or self.model) + saved_available = bool(self.api_key and _model_provider(selected_model) == "openai") + verified, message = ( + self._verify_saved_key(selected_model, refresh=refresh) + if saved_available + else (False, "Enter an API key for the selected provider") + ) + return { + "model": self.model, + "selected_model": selected_model, + "provider": _model_provider(selected_model), + "api_key_configured": saved_available, + "api_key_masked": "••••••••••••••••" if saved_available else "", + "api_key_verified": verified, + "verification_message": message, + } + + def verify_requested_key( + self, + *, + model: str, + api_key: str = "", + use_saved: bool = False, + refresh: bool = False, + ) -> tuple[bool, str]: + selected_model = validate_model_name(model) + if use_saved: + return self._verify_saved_key(selected_model, refresh=refresh) + return verify_model_key(api_key, selected_model) + + def _encrypt_credential(self, api_key: str) -> str: + return self.credential_cipher.encrypt(api_key.encode("utf-8")).decode("ascii") + + def _job_api_key(self, job: dict) -> str: + encrypted = str(job.get("credential_override", "")) + if encrypted: + try: + return self.credential_cipher.decrypt( + encrypted.encode("ascii"), ttl=31 * 24 * 60 * 60 + ).decode("utf-8") + except (InvalidToken, ValueError, TypeError) as exc: + raise RuntimeError("The job credential override is unavailable") from exc + if _model_provider(str(job.get("model", self.model))) == "openai": + return self.api_key + raise RuntimeError("No API key is available for the selected model") + @property def db(self): if self._db is None: @@ -193,12 +331,35 @@ def _job_ref(self, job_id: str): def internal_token_valid(self, supplied: str) -> bool: return bool(self.job_token and hmac.compare_digest(self.job_token, supplied or "")) - def create_job(self, *, base_url: str, email: str) -> dict: + def create_job( + self, + *, + base_url: str, + email: str, + model: str = "", + api_key: str = "", + ) -> dict: """Persist and enqueue a new whole-site audit job.""" if not self.configured: raise RuntimeError("Asynchronous site auditing is not configured") normalized_url = validate_public_url(str(base_url or "")) normalized_email = validate_request_email(email, self.allowed_domains) + selected_model = validate_model_name(model or self.model) + override_key = str(api_key or "").strip() + if override_key: + valid, message = verify_model_key(override_key, selected_model) + if not valid: + raise ValueError(message) + encrypted_override = self._encrypt_credential(override_key) + credential_source = "override" + else: + if _model_provider(selected_model) != "openai": + raise ValueError("Enter an API key when selecting a non-OpenAI model") + valid, message = self._verify_saved_key(selected_model) + if not valid: + raise ValueError(message) + encrypted_override = "" + credential_source = "saved" email_hash = _email_hash(normalized_email) recent_for_email = ( @@ -220,7 +381,8 @@ def create_job(self, *, base_url: str, email: str) -> dict: "email": normalized_email, "email_hash": email_hash, "status": "queued", - "model": self.model, + "model": selected_model, + "credential_source": credential_source, "max_pages": 200, "pages_total": 0, "pages_completed": 0, @@ -231,6 +393,8 @@ def create_job(self, *, base_url: str, email: str) -> dict: "updated_at": _now(), "expires_at": _now() + timedelta(days=30), } + if encrypted_override: + job["credential_override"] = encrypted_override self._job_ref(job_id).set(job) try: self.dispatcher.enqueue( @@ -295,8 +459,8 @@ def run_discovery(self, job_id: str) -> dict: job_ref.update({"status": "discovering", "updated_at": _now()}) discovery = discover_site_urls( job["base_url"], - api_key=self.api_key, - model=self.model, + api_key=self._job_api_key(job), + model=job["model"], max_pages=200, ) batch = self.db.batch() @@ -423,7 +587,11 @@ def run_page( try: html_content, final_url = fetch_public_html(page["url"]) - result = audit_callable(html_content, self.api_key, self.model) + result = audit_callable( + html_content, + self._job_api_key(job), + job["model"], + ) if not result.get("success"): raise RuntimeError(result.get("error") or "Page audit failed") csv_text = result.get("csv_report") or "" @@ -517,6 +685,13 @@ def finalize(self, job_id: str) -> dict: report.csv_bytes, content_type="text/csv; charset=utf-8" ) download_url = f"{self.service_url}/api/site-audits/{job_id}/report" + if job.get("credential_override"): + job_ref.update( + { + "credential_override": firestore.DELETE_FIELD, + "credential_cleared_at": _now(), + } + ) try: send_report_email( recipient=job["email"], From 3e54ce3191b4879e7f0b32bfdabf01f2d85e2108 Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 18:34:46 -0400 Subject: [PATCH 03/15] Harden site crawling email delivery and progress --- entry_points/api_server.py | 18 +++++ index.html | 71 ++++++++++++++++-- pyproject.toml | 1 + requirements.txt | 11 ++- styles.css | 5 +- tests/test_site_audit.py | 113 +++++++++++++++++++++++++++- uv.lock | 35 +++++++++ vision_aid/site_audit/crawler.py | 104 ++++++++++++++++++++++++-- vision_aid/site_audit/jobs.py | 124 ++++++++++++++++++++++++++++--- 9 files changed, 453 insertions(+), 29 deletions(-) diff --git a/entry_points/api_server.py b/entry_points/api_server.py index a2dccd8..fe4d7d3 100644 --- a/entry_points/api_server.py +++ b/entry_points/api_server.py @@ -304,6 +304,8 @@ def do_POST(self): # noqa: N802 self._handle_validate_key() elif self.path == "/api/site-audits": self._handle_create_site_audit() + elif re.fullmatch(r"/api/site-audits/[A-Za-z0-9_-]+/resend", self.path): + self._handle_site_audit_resend(self.path) else: self.send_error(404, "Not Found") @@ -737,6 +739,22 @@ def _handle_site_audit_report(self, path: str): self.end_headers() self.wfile.write(report) + def _handle_site_audit_resend(self, path: str): + job_id = path.split("/")[-2] + try: + job = get_coordinator().resend_report(job_id) + except ValueError as exc: + self._send_json({"success": False, "error": str(exc)}, 400) + return + except Exception as exc: + print(f" Site audit email resend failed: {exc}") + self._send_json( + {"success": False, "error": "The mail server did not accept the resend"}, + 503, + ) + return + self._send_json({"success": True, "job": job}) + def _handle_internal_site_audit(self, operation: str): """Handle authenticated callbacks from the dedicated Cloud Tasks queue.""" coordinator = get_coordinator() diff --git a/index.html b/index.html index 3591bcf..07c9316 100644 --- a/index.html +++ b/index.html @@ -1184,6 +1184,16 @@

Accessibility Audit Tool

} }); + function formatDuration(totalSeconds) { + const seconds = Math.max(0, Math.round(Number(totalSeconds) || 0)); + if (seconds < 60) return `${seconds} sec`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes} min`; + const hours = Math.floor(minutes / 60); + const remainder = minutes % 60; + return remainder ? `${hours} hr ${remainder} min` : `${hours} hr`; + } + function renderSiteJob(container, job) { const finished = (job.pages_completed || 0) + (job.pages_failed || 0); const total = job.pages_total || 0; @@ -1199,17 +1209,68 @@

Accessibility Audit Tool

const reportLink = job.report_url ? `

Download report ZIP

` : ""; + const details = []; + if (total) { + details.push(`${finished} of ${total} pages processed`); + details.push(`${job.pages_completed || 0} successful`); + if (job.pages_failed) details.push(`${job.pages_failed} failed`); + } else if (job.status === "discovering") { + details.push("Checking sitemaps, site links, CMS records, and web search…"); + } else { + details.push("Waiting to begin page discovery…"); + } + if (job.elapsed_seconds != null) { + details.push(`Elapsed: ${formatDuration(job.elapsed_seconds)}`); + } + if (job.estimated_seconds_remaining > 0) { + details.push(`Estimated remaining: ${formatDuration(job.estimated_seconds_remaining)}`); + } + if (job.candidate_count) { + details.push(`${job.candidate_count} public page candidates found`); + } + const deliveryNotice = job.email_delivery_status === "accepted" + ? `

Email submitted successfully. The mail server accepted it${job.email_accepted_at ? ` at ${esc(new Date(job.email_accepted_at).toLocaleString())}` : ""}.

` + : ""; + const resendButton = job.report_url + ? `

` + : ""; container.innerHTML = `
-

${esc(labels[job.status] || job.status)}

-
-

${total ? `${finished} of ${total} pages finished` : "Finding public pages\u2026"}

+

${esc(labels[job.status] || job.status)} — ${pct}%

+
+

${details.map(esc).join(" • ")}

You may close this page. The report will be emailed when complete.

${job.capped ? "

The site exceeded the limit; the audit is capped at 200 pages.

" : ""} - ${job.email_sent ? "

Email sent.

" : ""} + ${deliveryNotice} ${job.last_error ? `

${esc(job.last_error)}

` : ""} - ${reportLink}`; + ${reportLink} + ${resendButton}`; + const resend = container.querySelector(".resend-report-btn"); + if (resend) { + resend.addEventListener("click", async () => { + resend.disabled = true; + resend.textContent = "Submitting email…"; + try { + const response = await fetch( + `/api/site-audits/${encodeURIComponent(job.job_id)}/resend`, + { method: "POST" } + ); + const result = await response.json(); + if (!response.ok || !result.success) { + throw new Error(result.error || "Email resend failed"); + } + renderSiteJob(container, result.job); + } catch (error) { + resend.disabled = false; + resend.textContent = "Resend report email"; + const notice = document.createElement("p"); + notice.className = "audit-error"; + notice.textContent = error.message; + resend.parentElement.after(notice); + } + }); + } } async function pollSiteJob(container, jobId) { diff --git a/pyproject.toml b/pyproject.toml index e734905..41628ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "google-cloud-tasks>=2.19.0", "python-dotenv~=1.2.1", "requests>=2.32.2", + "curl-cffi>=0.14.0", ] [tool.setuptools] diff --git a/requirements.txt b/requirements.txt index 5314bf1..67ae045 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,11 +15,14 @@ beautifulsoup4==4.15.0 # via visionaid certifi==2026.7.22 # via + # curl-cffi # httpcore # httpx # requests -cffi==2.1.1 ; platform_python_implementation != 'PyPy' - # via cryptography +cffi==2.1.1 + # via + # cryptography + # curl-cffi charset-normalizer==3.4.9 # via requests colorama==0.4.6 ; sys_platform == 'win32' @@ -28,6 +31,8 @@ cryptography==50.0.0 # via # google-auth # visionaid +curl-cffi==0.16.2 + # via visionaid distro==1.9.0 # via # anthropic @@ -124,7 +129,7 @@ pyasn1==0.6.4 # via pyasn1-modules pyasn1-modules==0.4.2 # via google-auth -pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' +pycparser==3.0 ; implementation_name != 'PyPy' # via cffi pydantic==2.13.4 # via diff --git a/styles.css b/styles.css index ef35265..e32a5b3 100644 --- a/styles.css +++ b/styles.css @@ -1445,7 +1445,8 @@ section:nth-child(even) { .progress-detail { font-size: 0.85rem; color: #666; - word-break: break-all; + line-height: 1.6; + overflow-wrap: anywhere; } /* ============================================ @@ -1540,4 +1541,4 @@ section:nth-child(even) { display: flex; align-items: center; gap: 8px; -} \ No newline at end of file +} diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index 4fa2d28..ba8a4bf 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -7,6 +7,7 @@ import os import unittest import zipfile +from datetime import datetime, timedelta, timezone from unittest import mock from cryptography.fernet import Fernet @@ -18,9 +19,14 @@ _is_candidate_url, _sitemap_locations, canonicalize_url, + fetch_public_html, validate_public_url, ) -from vision_aid.site_audit.jobs import SiteAuditCoordinator, validate_request_email +from vision_aid.site_audit.jobs import ( + SiteAuditCoordinator, + send_report_email, + validate_request_email, +) from vision_aid.site_audit.report import build_site_report @@ -119,6 +125,30 @@ def test_ai_order_cannot_drop_discovered_pages(self): self.assertEqual(ordered[0].url, "https://example.com/about/") self.assertEqual({item.url for item in ordered}, {item.url for item in candidates}) + def test_browser_fingerprint_fallback_is_used_when_standard_request_is_blocked(self): + blocked = mock.Mock(status_code=403) + blocked.close = mock.Mock() + session = mock.Mock() + session.get.return_value = blocked + browser_response = mock.Mock( + status_code=200, + is_redirect=False, + is_permanent_redirect=False, + headers={"Content-Type": "text/html; charset=utf-8"}, + encoding="utf-8", + ) + browser_response.iter_content.return_value = [b"working"] + browser_response.raise_for_status.return_value = None + with mock.patch( + "curl_cffi.requests.get", return_value=browser_response + ) as browser_get: + html, final_url = fetch_public_html( + "https://example.com/", session=session + ) + self.assertEqual(html, "working") + self.assertEqual(final_url, "https://example.com/") + browser_get.assert_called_once() + class EmailTests(unittest.TestCase): def test_allowed_domain(self): @@ -131,6 +161,65 @@ def test_disallowed_domain(self): with self.assertRaisesRegex(ValueError, "limited"): validate_request_email("person@example.com", {"visionaid.org"}) + def test_email_records_smtp_acceptance_and_omits_zip_by_default(self): + smtp = mock.Mock() + smtp.send_message.return_value = {} + smtp_context = mock.Mock() + smtp_context.__enter__ = mock.Mock(return_value=smtp) + smtp_context.__exit__ = mock.Mock(return_value=False) + with mock.patch.dict( + os.environ, + { + "SMTP_HOST": "smtp.example.com", + "SMTP_PORT": "587", + "SMTP_USER": "sender@example.com", + "SMTP_PASSWORD": "secret", + "SMTP_FROM": "sender@example.com", + "DAT_EMAIL_ATTACH_REPORT": "false", + }, + ), mock.patch( + "vision_aid.site_audit.jobs.smtplib.SMTP", return_value=smtp_context + ): + receipt = send_report_email( + recipient="recipient@example.com", + base_url="https://example.com/", + pages=3, + findings=7, + download_url="https://dat.example.com/report", + report_zip=b"zip", + ) + message = smtp.send_message.call_args.args[0] + self.assertEqual(list(message.iter_attachments()), []) + self.assertEqual(receipt["recipient"], "recipient@example.com") + self.assertFalse(receipt["attachment_included"]) + self.assertTrue(receipt["message_id"].startswith("<")) + + def test_email_raises_when_recipient_is_refused(self): + smtp = mock.Mock() + smtp.send_message.return_value = {"recipient@example.com": (550, b"refused")} + smtp_context = mock.Mock() + smtp_context.__enter__ = mock.Mock(return_value=smtp) + smtp_context.__exit__ = mock.Mock(return_value=False) + with mock.patch.dict( + os.environ, + { + "SMTP_HOST": "smtp.example.com", + "SMTP_USER": "sender@example.com", + "SMTP_PASSWORD": "secret", + "SMTP_FROM": "sender@example.com", + }, + ), mock.patch( + "vision_aid.site_audit.jobs.smtplib.SMTP", return_value=smtp_context + ), self.assertRaisesRegex(RuntimeError, "refused"): + send_report_email( + recipient="recipient@example.com", + base_url="https://example.com/", + pages=1, + findings=0, + download_url="https://dat.example.com/report", + report_zip=b"zip", + ) + class CredentialTests(unittest.TestCase): def test_override_is_encrypted_and_decrypted_only_for_the_job(self): @@ -176,6 +265,28 @@ def test_public_config_masks_and_never_returns_saved_key(self): self.assertNotIn("saved-test-credential", repr(config)) +class ProgressTests(unittest.TestCase): + def test_public_job_reports_percentage_elapsed_time_and_eta(self): + now = datetime(2026, 8, 29, 18, 0, tzinfo=timezone.utc) + coordinator = SiteAuditCoordinator() + coordinator.service_url = "https://dat.example.com" + job = { + "job_id": "test-job", + "status": "auditing", + "pages_total": 10, + "pages_completed": 4, + "pages_failed": 0, + "created_at": now - timedelta(minutes=4), + "audit_started_at": now - timedelta(minutes=2), + } + with mock.patch("vision_aid.site_audit.jobs._now", return_value=now): + public = coordinator.public_job(job) + self.assertEqual(public["progress_percent"], 39) + self.assertEqual(public["pages_remaining"], 6) + self.assertEqual(public["elapsed_seconds"], 120) + self.assertEqual(public["estimated_seconds_remaining"], 180) + + class ReportTests(unittest.TestCase): def test_report_zip_contains_cover_csv_and_summary(self): csv_text = ( diff --git a/uv.lock b/uv.lock index 736db6f..704237d 100644 --- a/uv.lock +++ b/uv.lock @@ -306,6 +306,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/b5/c2c5fce26f0ee40d21bafe7f191d29a34b35a65ac4fe8a1191d1983612e9/cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9", size = 3813796, upload-time = "2026-07-31T14:25:02.298Z" }, ] +[[package]] +name = "curl-cffi" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/f6/347067dfacb19e44a4166d7bdb183e3a2629680beceb5e52f7cb2cc1a3b4/curl_cffi-0.16.2.tar.gz", hash = "sha256:2986a86cdcf514ab73632c2de62a01db3cc97f7ecf17798a1be16180f4474198", size = 238986, upload-time = "2026-08-25T11:51:11.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/ac/3772115da1b6b9b6064e8c128dfeb6c9e708dd4603a17eb52f658558c5e1/curl_cffi-0.16.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:afe05fc1022039c2853177ef13f67b7a9915bff62e35eb4fa1a295166bc55a34", size = 3024444, upload-time = "2026-08-25T11:50:32.215Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/57a757acb1b43ef944735835b50cfdf00150113164d755a99f8d21d875f3/curl_cffi-0.16.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e6f238ddfef41a5d074bcc2f6b8fc21e653a1e7d4d9a2163839030d9de24b2ca", size = 2781420, upload-time = "2026-08-25T11:50:33.851Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8c/755116d4f57fbad64947f47141ccc342a30bf5998ca6a33a2bcc33f34ab9/curl_cffi-0.16.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5382ca2a67a459dd6f64203555fd248b71abe4ca6cd40669902916dc5f558a33", size = 12827642, upload-time = "2026-08-25T11:50:35.388Z" }, + { url = "https://files.pythonhosted.org/packages/8c/31/aedc88071ee74b9b7848e48f11ddff1c242ab52f914f47309e0d15c4982f/curl_cffi-0.16.2-cp310-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b0deed58ca724ffdb80e1f4029208325b2a4999f0a438f55037c591232e6c18", size = 12651309, upload-time = "2026-08-25T11:50:37.272Z" }, + { url = "https://files.pythonhosted.org/packages/65/b0/b3dd929480f419f8aa921860afb44e15d2a22fd0be5c545296b9db865aa3/curl_cffi-0.16.2-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b2a59707f5345491a08a3d45f6895b1c9dc192d403a1ebc42c1d1ef43a58453e", size = 13474592, upload-time = "2026-08-25T11:50:39.344Z" }, + { url = "https://files.pythonhosted.org/packages/8c/14/fbed6244544be92659de5cad4cca8d0ef79a05c17518d1b3fd2ca84d80cc/curl_cffi-0.16.2-cp310-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3645b13dc1d7dd727e90e847a767b6a2fb5a16a0cbc86a1fb8e77714775f498", size = 12833348, upload-time = "2026-08-25T11:50:41.395Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8d/ee9b29ad9e2788cce9be333caac445be3950780e2eb1bdc47b53bb1f06c6/curl_cffi-0.16.2-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b07b19225a979ff85f30868712d8a274e69f013489c5b40102ec0167cd0a756a", size = 12604773, upload-time = "2026-08-25T11:50:43.476Z" }, + { url = "https://files.pythonhosted.org/packages/60/b5/373754bf9bbdac84ec0d668d9f75487ed9ca236802c3c1edb9bb34cb045f/curl_cffi-0.16.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8b6877cf7a1470e357441d5fbd4a338d2372d49c352f0b8a57e565910f932f4d", size = 12580793, upload-time = "2026-08-25T11:50:45.681Z" }, + { url = "https://files.pythonhosted.org/packages/c4/61/7f659fe76575494268af7ac622250f3841b2a43eac8207d1758e61e96d05/curl_cffi-0.16.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:23f490b0b11d87beb94d5d26c4661c26f0439396f839af322a1847c268971913", size = 13240951, upload-time = "2026-08-25T11:50:48.388Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ac/21e1241211aa53102417c6952bb2929b1d7fee7806c9f1745b2d9f93ccdf/curl_cffi-0.16.2-cp310-abi3-win_amd64.whl", hash = "sha256:3614728ce7f15507b1e72761639c0524a434c3c8b141eb1aae7e8e037dcb292b", size = 1977254, upload-time = "2026-08-25T11:50:50.452Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d8/178f229cd6169df842e52aa54b9b523b9642f7e09ca4f7a646a9b096a3ec/curl_cffi-0.16.2-cp310-abi3-win_arm64.whl", hash = "sha256:cbf7d2d14057bc62ae1da9e03f18f08ca18dbb43c7fca1eaa827f5d703b32f5c", size = 1711623, upload-time = "2026-08-25T11:50:52.034Z" }, + { url = "https://files.pythonhosted.org/packages/83/0b/97ef41ea98548f48dff99feddea318cfc9266f15c891664bcc88fd5d6b7c/curl_cffi-0.16.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:58598186eccd24d2b2e126f945d8a5bdca0066a2789052023d3d8370ebadca30", size = 8605929, upload-time = "2026-08-25T11:50:53.748Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e8/66da94f9183849f901402ebe3c7365bb2541e7109779774ff17741fea1c5/curl_cffi-0.16.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:16f30cf2f08f33a74ade44770cb6efc0eaacfbdb2a36e18a7817f9ff2c2025a1", size = 8605614, upload-time = "2026-08-25T11:50:55.593Z" }, + { url = "https://files.pythonhosted.org/packages/df/7e/a63ae4b566f9cd186d609662f03d8599bf0001808dba2059e1e780534f2e/curl_cffi-0.16.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ec5944bf5e4cc5dd1de35e86ffe419db59df7335aa868e278d7f0e848f4cc253", size = 3025009, upload-time = "2026-08-25T11:50:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/c3/49/f8f5f0713e2a06242cffce2ef2dacc2084c01c7cde05a31600c737e439ed/curl_cffi-0.16.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9a45612bfe50d6d5f7825a9a806b044969281bef8928e42ffae503d58cfe6b65", size = 2781587, upload-time = "2026-08-25T11:50:59.039Z" }, + { url = "https://files.pythonhosted.org/packages/22/b9/efdd5c44d36474a811fe4304fe7daa7612804bc69daf050a223720926a33/curl_cffi-0.16.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e5ef2ce4044f432e188d4338d1826d1d70f9f38f5a07e78a88592c9d133c4c3a", size = 12832014, upload-time = "2026-08-25T11:51:00.598Z" }, + { url = "https://files.pythonhosted.org/packages/83/86/9b92aac7365a5894fa4ee253c52ff0a5924b233341c70181cb926a071057/curl_cffi-0.16.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e34b4158fcd0192e38bb8420377ad4be91e62d5ea52848f7a5e9e07579ad29ac", size = 13483396, upload-time = "2026-08-25T11:51:02.678Z" }, + { url = "https://files.pythonhosted.org/packages/1c/31/cdb7e2899e1a6ec62a1f920f1550cf99f9c929b36d800e1502199cd2fd12/curl_cffi-0.16.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:58f346db1a203c709d99619322f6c084d30a47ac2495941f2c04785a5441e1ad", size = 12587394, upload-time = "2026-08-25T11:51:04.831Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c2/b1358b61ee061ea413a014465d5437d00a9d59b654684857dc2b88391d6e/curl_cffi-0.16.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7314f25d2840cba696ecebe8a36be8d62e9f2106cb4e8aa380e6f6f0a9fc1ef2", size = 13249643, upload-time = "2026-08-25T11:51:06.82Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bb/7209914870ac5e97b1a19ecca492ac7632a61ffee039284039f7b97c2599/curl_cffi-0.16.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d7effcc9dfd3aee1d8871efc8fb397041b5546d0af1653a5af26e845dd194ff3", size = 2030216, upload-time = "2026-08-25T11:51:08.909Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0c/77b89a0efe4a23a9b370f049ad4988a57232c2eb82b3577a6e1d8fe0e597/curl_cffi-0.16.2-cp314-cp314t-win_arm64.whl", hash = "sha256:6bc754ad2dcb5287a157f34d615dde442e80e688289caedc953be8d92fe3f50b", size = 1779425, upload-time = "2026-08-25T11:51:10.4Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1115,6 +1148,7 @@ dependencies = [ { name = "anthropic" }, { name = "beautifulsoup4" }, { name = "cryptography" }, + { name = "curl-cffi" }, { name = "google-cloud-firestore" }, { name = "google-cloud-storage" }, { name = "google-cloud-tasks" }, @@ -1130,6 +1164,7 @@ requires-dist = [ { name = "anthropic", specifier = "~=0.84.0" }, { name = "beautifulsoup4", specifier = ">=4.9.2" }, { name = "cryptography", specifier = ">=45.0.0" }, + { name = "curl-cffi", specifier = ">=0.14.0" }, { name = "google-cloud-firestore", specifier = ">=2.21.0" }, { name = "google-cloud-storage", specifier = ">=3.3.0" }, { name = "google-cloud-tasks", specifier = ">=2.19.0" }, diff --git a/vision_aid/site_audit/crawler.py b/vision_aid/site_audit/crawler.py index 9c722a9..03ec374 100644 --- a/vision_aid/site_audit/crawler.py +++ b/vision_aid/site_audit/crawler.py @@ -130,7 +130,9 @@ class DiscoveryResult: candidate_count: int capped: bool ai_used: bool + ai_web_used: bool sitemap_count: int + source_counts: dict[str, int] def canonicalize_url(url: str) -> str: @@ -202,13 +204,37 @@ def _request( """Fetch a public URL safely and return ``(body, final_url, content_type)``.""" current = validate_public_url(url) for _ in range(6): - response = session.get( - current, - headers=REQUEST_HEADERS, - timeout=timeout, - allow_redirects=False, - stream=True, - ) + try: + response = session.get( + current, + headers=REQUEST_HEADERS, + timeout=timeout, + allow_redirects=False, + stream=True, + ) + if response.status_code in {403, 406, 429, 503}: + response.close() + raise requests.HTTPError( + f"HTTP {response.status_code} blocked the standard crawler" + ) + except (requests.RequestException, OSError) as primary_error: + # Some bot-protected sites reject the default Python TLS signature + # even when the headers identify a current browser. curl_cffi uses a + # real browser TLS/HTTP fingerprint while the redirect and SSRF + # checks below remain under our control. + try: + from curl_cffi import requests as browser_requests + + response = browser_requests.get( + current, + headers=REQUEST_HEADERS, + timeout=timeout, + allow_redirects=False, + impersonate="chrome", + stream=True, + ) + except Exception: + raise primary_error if response.is_redirect or response.is_permanent_redirect: location = response.headers.get("Location") response.close() @@ -477,6 +503,45 @@ def _ai_order_candidates( return [candidates[index] for index in indexes] +def _ai_web_discover_candidates( + base_url: str, + *, + api_key: str, + model: str, + max_pages: int, +) -> list[str]: + """Use OpenAI web search only when the target blocks every direct source.""" + if not api_key or not model.startswith(("gpt-", "o1", "o3", "o4")): + return [] + + from openai import OpenAI + + host = urlparse(base_url).hostname or base_url + client = OpenAI(api_key=api_key, max_retries=4, timeout=180.0) + response = client.responses.create( + model=model, + tools=[{"type": "web_search"}], + reasoning={"effort": "low"}, + max_output_tokens=12_000, + input=f"""Find the public HTML pages belonging to {base_url} for a complete +accessibility audit. Search the web for indexed pages on site:{host}, inspect +public sitemap results when available, and return up to {max_pages} canonical +same-site URLs. Exclude files, feeds, login/admin pages, previews, search-result +pages, and action URLs. Do not invent URLs. Return JSON only as +{{"urls":["https://..."]}}. +""", + ) + parsed = _parse_json_object(response.output_text) + urls: list[str] = [] + for value in parsed.get("urls", []): + candidate = canonicalize_url(str(value)) + if _is_candidate_url(candidate, base_url) and candidate not in urls: + urls.append(candidate) + if len(urls) >= max_pages: + break + return urls + + def discover_site_urls( base_url: str, *, @@ -500,7 +565,21 @@ def discover_site_urls( sitemap_urls, sitemap_count = _read_sitemaps(normalized_base, active_session) wordpress_urls = _discover_wordpress_urls(normalized_base, active_session) - queue = deque([normalized_base, *sitemap_urls, *wordpress_urls]) + ai_web_urls: list[str] = [] + ai_web_used = False + if len({normalized_base, *sitemap_urls, *wordpress_urls}) <= 1: + try: + ai_web_urls = _ai_web_discover_candidates( + normalized_base, + api_key=api_key, + model=model, + max_pages=max_pages, + ) + ai_web_used = bool(ai_web_urls) + except Exception as exc: + print(f" WARNING: AI web discovery failed ({exc}); continuing direct crawl") + + queue = deque([normalized_base, *sitemap_urls, *wordpress_urls, *ai_web_urls]) candidates: dict[str, PageCandidate] = { normalized_base: PageCandidate(normalized_base, source="base") } @@ -512,6 +591,10 @@ def discover_site_urls( candidates.setdefault( wordpress_url, PageCandidate(wordpress_url, source="cms") ) + for ai_web_url in ai_web_urls: + candidates.setdefault( + ai_web_url, PageCandidate(ai_web_url, source="ai-web") + ) fetched: set[str] = set() while queue and len(fetched) < DISCOVERY_FETCH_LIMIT and len(candidates) < CANDIDATE_LIMIT: @@ -562,10 +645,15 @@ def discover_site_urls( if normalized_base not in {item.url for item in selected}: selected = [base_candidate, *selected][:max_pages] capped = len(ordered) > max_pages + source_counts: dict[str, int] = {} + for candidate in ordered: + source_counts[candidate.source] = source_counts.get(candidate.source, 0) + 1 return DiscoveryResult( pages=selected[:max_pages], candidate_count=len(ordered), capped=capped, ai_used=ai_used, + ai_web_used=ai_web_used, sitemap_count=sitemap_count, + source_counts=source_counts, ) diff --git a/vision_aid/site_audit/jobs.py b/vision_aid/site_audit/jobs.py index ad8f779..948c2a0 100644 --- a/vision_aid/site_audit/jobs.py +++ b/vision_aid/site_audit/jobs.py @@ -15,6 +15,7 @@ import urllib.request from datetime import datetime, timedelta, timezone from email.message import EmailMessage +from email.utils import format_datetime, make_msgid from typing import Callable from urllib.parse import quote, urlparse @@ -153,8 +154,8 @@ def send_report_email( findings: int, download_url: str, report_zip: bytes, -) -> None: - """Send the completed site report using deployment SMTP settings.""" +) -> dict: + """Submit a completed report to SMTP and return its acceptance receipt.""" smtp_host = os.getenv("SMTP_HOST", "").strip() smtp_port = int(os.getenv("SMTP_PORT", "587")) smtp_user = os.getenv("SMTP_USER", "").strip() @@ -168,14 +169,23 @@ def send_report_email( message["Subject"] = f"DAT whole-site accessibility report: {host}" message["From"] = smtp_from message["To"] = recipient + message["Date"] = format_datetime(_now()) + message_id = make_msgid(domain=(smtp_from.rsplit("@", 1)[-1] or None)) + message["Message-ID"] = message_id + attach_report = os.getenv("DAT_EMAIL_ATTACH_REPORT", "").strip().lower() in { + "1", + "true", + "yes", + } message.set_content( "Your Vision Aid DAT whole-site accessibility report is ready.\n\n" f"Site: {base_url}\nPages processed: {pages}\nFindings: {findings}\n" f"Download: {download_url}\n\n" - "The attached ZIP contains a printable HTML report, a CSV findings file, " - "and a JSON page summary." + "For reliable delivery, the report is provided through the secure download " + "link instead of as an email attachment. Sign in with the DAT testing " + "password if prompted." ) - if len(report_zip) <= 18 * 1024 * 1024: + if attach_report and len(report_zip) <= 18 * 1024 * 1024: message.add_attachment( report_zip, maintype="application", @@ -188,7 +198,22 @@ def send_report_email( smtp.starttls() smtp.ehlo() smtp.login(smtp_user, smtp_password) - smtp.send_message(message) + refused = smtp.send_message( + message, + from_addr=smtp_from, + to_addrs=[recipient], + ) + if refused: + refused_recipients = ", ".join(str(item) for item in refused) + raise RuntimeError(f"Mail server refused: {refused_recipients}") + accepted_at = _now() + return { + "message_id": message_id, + "accepted_at": accepted_at, + "recipient": recipient, + "smtp_host": smtp_host, + "attachment_included": attach_report and len(report_zip) <= 18 * 1024 * 1024, + } class SiteAuditCoordinator: @@ -427,17 +452,48 @@ def public_job(self, job: dict) -> dict: "pages_completed", "pages_failed", "candidate_count", + "discovery_sources", "capped", "total_findings", "report_ready", "email_sent", + "email_delivery_status", + "email_accepted_at", "last_error", + "created_at", + "audit_started_at", + "completed_at", ) if key in job } total = int(job.get("pages_total") or 0) finished = int(job.get("pages_completed") or 0) + int(job.get("pages_failed") or 0) - result["progress_percent"] = round((finished / total) * 100) if total else 0 + remaining = max(0, total - finished) + status = str(job.get("status", "")) + if status == "complete": + progress = 100 + elif status == "finalizing": + progress = 98 + elif total: + progress = min(97, round((finished / total) * 97)) + elif status == "discovering": + progress = 2 + else: + progress = 0 + result["progress_percent"] = progress + result["pages_remaining"] = remaining + + started_at = job.get("audit_started_at") or job.get("created_at") + ended_at = job.get("completed_at") if status == "complete" else _now() + if isinstance(started_at, datetime) and isinstance(ended_at, datetime): + elapsed = max(0, int((ended_at - started_at).total_seconds())) + result["elapsed_seconds"] = elapsed + if status == "auditing" and finished > 0 and remaining > 0: + result["estimated_seconds_remaining"] = max( + 1, round((elapsed / finished) * remaining) + ) + elif status in {"finalizing", "complete"}: + result["estimated_seconds_remaining"] = 0 if job.get("status") == "complete": result["report_url"] = f"{self.service_url}/api/site-audits/{job['job_id']}/report" for key, value in list(result.items()): @@ -456,7 +512,13 @@ def run_discovery(self, job_id: str) -> dict: existing_pages = list(job_ref.collection("pages").stream()) if not existing_pages: - job_ref.update({"status": "discovering", "updated_at": _now()}) + job_ref.update( + { + "status": "discovering", + "discovery_started_at": _now(), + "updated_at": _now(), + } + ) discovery = discover_site_urls( job["base_url"], api_key=self._job_api_key(job), @@ -484,7 +546,9 @@ def run_discovery(self, job_id: str) -> dict: "candidate_count": discovery.candidate_count, "capped": discovery.capped, "ai_discovery_used": discovery.ai_used, + "ai_web_discovery_used": discovery.ai_web_used, "sitemap_count": discovery.sitemap_count, + "discovery_sources": discovery.source_counts, "updated_at": _now(), } ) @@ -498,7 +562,11 @@ def run_discovery(self, job_id: str) -> dict: {"job_id": job_id, "page_id": snapshot.id}, f"{job_id}-page-{index:03d}", ) - job_ref.update({"status": "auditing", "updated_at": _now()}) + refreshed_job = self.get_job(job_id) or {} + audit_update = {"status": "auditing", "updated_at": _now()} + if not refreshed_job.get("audit_started_at"): + audit_update["audit_started_at"] = _now() + job_ref.update(audit_update) return self.public_job(self.get_job(job_id)) def _store_page_result(self, job_id: str, page_id: str, payload: dict) -> str: @@ -556,6 +624,7 @@ def update(transaction): "pages_completed": completed, "pages_failed": failed, "updated_at": _now(), + "last_page_completed_at": _now(), } ready = completed + failed >= int(job.get("pages_total", 0)) if ready: @@ -693,7 +762,7 @@ def finalize(self, job_id: str) -> dict: } ) try: - send_report_email( + delivery = send_report_email( recipient=job["email"], base_url=job["base_url"], pages=len(pages), @@ -714,6 +783,10 @@ def finalize(self, job_id: str) -> dict: "report_objects": objects, "total_findings": report.total_findings, "email_sent": True, + "email_delivery_status": "accepted", + "email_message_id": delivery["message_id"], + "email_accepted_at": delivery["accepted_at"], + "email_attachment_included": delivery["attachment_included"], "completed_at": _now(), "updated_at": _now(), "last_error": firestore.DELETE_FIELD, @@ -721,6 +794,37 @@ def finalize(self, job_id: str) -> dict: ) return self.public_job(self.get_job(job_id)) + def resend_report(self, job_id: str) -> dict: + """Resubmit an existing completed report without rerunning the audit.""" + job_ref = self._job_ref(job_id) + job = self.get_job(job_id) + if not job or not job.get("report_ready"): + raise ValueError("The report is not ready to resend") + report_zip = self.report_bytes(job_id) + if report_zip is None: + raise ValueError("The report file is unavailable") + delivery = send_report_email( + recipient=job["email"], + base_url=job["base_url"], + pages=int(job.get("pages_total", 0)), + findings=int(job.get("total_findings", 0)), + download_url=f"{self.service_url}/api/site-audits/{job_id}/report", + report_zip=report_zip, + ) + job_ref.update( + { + "email_sent": True, + "email_delivery_status": "accepted", + "email_message_id": delivery["message_id"], + "email_accepted_at": delivery["accepted_at"], + "email_attachment_included": delivery["attachment_included"], + "email_resend_count": firestore.Increment(1), + "updated_at": _now(), + "last_error": firestore.DELETE_FIELD, + } + ) + return self.public_job(self.get_job(job_id)) + def report_bytes(self, job_id: str) -> bytes | None: job = self.get_job(job_id) if not job or not job.get("report_ready"): From 543f78ce931d057250ed6a007aa99435321b096f Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 18:42:25 -0400 Subject: [PATCH 04/15] Handle JavaScript anti-bot crawl challenges --- Dockerfile | 10 ++- pyproject.toml | 1 + requirements.txt | 7 ++ tests/test_site_audit.py | 35 ++++++++++ uv.lock | 110 +++++++++++++++++++++++++++++++ vision_aid/site_audit/crawler.py | 106 +++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 555501e..d318c4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,15 @@ RUN uv sync --frozen --no-dev FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + PYTHONDONTWRITEBYTECODE=1 \ + DAT_CHROMIUM_EXECUTABLE=/usr/bin/chromium + +# SiteGround and similar CDNs can return JavaScript security interstitials to +# cloud data-center traffic. Chromium executes the legitimate public-site +# challenge so the auditor receives the requested page, never the interstitial. +RUN apt-get update \ + && apt-get install -y --no-install-recommends chromium \ + && rm -rf /var/lib/apt/lists/* # Single chowned COPY: the virtualenv lands in exactly one layer. Copying it # after creating it, rather than chown -R'ing it in place, keeps the image from diff --git a/pyproject.toml b/pyproject.toml index 41628ac..cb28c42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "python-dotenv~=1.2.1", "requests>=2.32.2", "curl-cffi>=0.14.0", + "playwright>=1.55.0", ] [tool.setuptools] diff --git a/requirements.txt b/requirements.txt index 67ae045..a95f198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,6 +77,8 @@ googleapis-common-protos==1.75.2 # google-api-core # grpc-google-iam-v1 # grpcio-status +greenlet==3.5.5 + # via playwright grpc-google-iam-v1==0.14.5 # via google-cloud-tasks grpcio==1.83.1 @@ -111,6 +113,8 @@ lxml==6.1.1 # via visionaid openai==2.53.0 # via visionaid +playwright==1.62.0 + # via visionaid proto-plus==1.28.4 # via # google-api-core @@ -138,6 +142,8 @@ pydantic==2.13.4 # openai pydantic-core==2.46.4 # via pydantic +pyee==13.0.1 + # via playwright python-dotenv==1.2.2 # via visionaid requests==2.34.2 @@ -168,6 +174,7 @@ typing-extensions==4.16.0 # openai # pydantic # pydantic-core + # pyee # typing-inspection typing-inspection==0.4.2 # via pydantic diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index ba8a4bf..cf4f877 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -17,6 +17,7 @@ _ai_order_candidates, _discover_wordpress_urls, _is_candidate_url, + _looks_like_bot_challenge, _sitemap_locations, canonicalize_url, fetch_public_html, @@ -149,6 +150,40 @@ def test_browser_fingerprint_fallback_is_used_when_standard_request_is_blocked(s self.assertEqual(final_url, "https://example.com/") browser_get.assert_called_once() + def test_siteground_soft_200_challenge_is_detected(self): + self.assertTrue( + _looks_like_bot_challenge( + "Robot Challenge ScreenChecking the site connection security", + "https://example.com/", + ) + ) + + def test_soft_200_challenge_is_replaced_by_browser_content(self): + challenge = mock.Mock( + status_code=200, + is_redirect=False, + is_permanent_redirect=False, + headers={"Content-Type": "text/html"}, + encoding="utf-8", + ) + challenge.iter_content.return_value = [ + b"Robot Challenge ScreenChecking the site connection security" + ] + challenge.raise_for_status.return_value = None + session = mock.Mock() + session.get.return_value = challenge + with mock.patch( + "vision_aid.site_audit.crawler._browser_fetch", + return_value=( + "Real page", + "https://example.com/", + "text/html", + ), + ) as browser_fetch: + html, _ = fetch_public_html("https://example.com/", session=session) + self.assertIn("Real page", html) + browser_fetch.assert_called_once() + class EmailTests(unittest.TestCase): def test_allowed_domain(self): diff --git a/uv.lock b/uv.lock index 704237d..83feaaf 100644 --- a/uv.lock +++ b/uv.lock @@ -541,6 +541,83 @@ grpc = [ { name = "grpcio" }, ] +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/a3/07297917485ee2ca85bc3c8dc6ed85ad3fffcf424047fba62671dba68e97/greenlet-3.5.5-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:be63afcbbccfad3dd95a1ba12ada84dab2ef32031973d80b5b92df67fa763a61", size = 294165, upload-time = "2026-08-10T13:25:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/6f732f9314cda54c5fd48a7620c7160f4f286967e8045ad94b9d66ce80b7/greenlet-3.5.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a268024ce2d7d2b04694bf1594058981a9fa663d1df4b762dee499211ed7c1c", size = 613610, upload-time = "2026-08-10T14:14:33.829Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c0/b27589e25d220289edcd4d582b2b17b83058d1a56d53d971b6ea1a34f10d/greenlet-3.5.5-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35cbb8bf55ace57fbccb4fb8622c4521713acd8691e77f4696d416ea7ca527da", size = 625481, upload-time = "2026-08-10T14:27:23.647Z" }, + { url = "https://files.pythonhosted.org/packages/39/82/5c873dbb4fb001d22fbbd50e80d4c1b0181ddae106856132160f84b94e88/greenlet-3.5.5-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:abc8bc8d9f935cd685457545b6a53863a877fdc12c2c0f5ee9beee18d9db139c", size = 633329, upload-time = "2026-08-10T14:30:06.062Z" }, + { url = "https://files.pythonhosted.org/packages/51/2d/f2c928218ac52f26d7a2c188c171d1b7e728b23782cb3347e7b4fce1493a/greenlet-3.5.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cc6df89ec5302337adc9cf096221cbed2510fd444b0e0f1586cf0470740864", size = 624562, upload-time = "2026-08-10T13:40:48.064Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/f7df98e72a8eb4a7edfec5a08d6d1a4ab53a52c95ba0b2ea6c10b8dd9bd0/greenlet-3.5.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:3134291427bb0f3526e9d90311988caf336eb43730e95244997a4fb15f45144f", size = 428145, upload-time = "2026-08-10T14:30:01.071Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4a/92fc51d5d35912f4f06eec037ba347985defd0be47463a010a325634d9d2/greenlet-3.5.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d9b454c5fc48aeaa7c4337813dbf513a6870468e426438a04d922c6d0fe63db", size = 1584909, upload-time = "2026-08-10T14:15:04.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/58/ed98b80ac5738c149a5258544843c45601ade1fd70f61740cdaead6351b3/greenlet-3.5.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03551ed792cb1b4fc0277a0c60dfd8c343894a0ba06fe60dcd22f568b433da39", size = 1651184, upload-time = "2026-08-10T13:40:28.879Z" }, + { url = "https://files.pythonhosted.org/packages/d8/be/b582ceb80cefdf9d8da34078714e4b12b3d16f509dee0f65e40a5cc8fc7d/greenlet-3.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:ab3df3dffb58bf70564e93a5cec7941e4d9faa5a36cc4234a10d3131afe04f53", size = 323280, upload-time = "2026-08-10T13:26:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/4d/18/5313c4c58598c38b0373c013e4ff2b3e6d258aaaa338f373335ebecdaddd/greenlet-3.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:2b70a766135540c472ac1393d57c2e1b4a2eb85bf526a1e41e6d096173a8cee5", size = 307785, upload-time = "2026-08-10T13:28:34.874Z" }, + { url = "https://files.pythonhosted.org/packages/2e/7e/9ecd0285e3153532ae07aeb88063c43c72b4221cf0d4d123b02f3682e3ff/greenlet-3.5.5-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:49520f0c95a48b42cf55414b8e8479beb274ea70431afc33e3f79903c71f4380", size = 295809, upload-time = "2026-08-10T13:25:34.023Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/60e4bbcc89252037b18087f2ec16405d5b2d5be42dde191bbf3667e96102/greenlet-3.5.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55272212cbc5f43d1d723725ab931f1939969b7e9523882ca58b55061769d053", size = 611910, upload-time = "2026-08-10T14:14:35.18Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/cd5134be659cd4a443e7a61ae670dabec165a814c51162916d637b6dd38e/greenlet-3.5.5-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655bca754a2ef4efcb0eb48a94d3f4593536d0f3d48f8ed44343c01d16a92f95", size = 624198, upload-time = "2026-08-10T14:27:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/9b/30/87c212b5c684d0e72974f1063b7a9687631e8985902c06e1016542c874e7/greenlet-3.5.5-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ca5d6ae0739e5764f2cfcfaa562ac5a990cbdaedca93251c5e3cf07c362371f", size = 629504, upload-time = "2026-08-10T14:30:07.967Z" }, + { url = "https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:147b25a42e5ca5be3d42356e8f608b37af715a1c196e9bf9d1627f3341adfe1d", size = 621439, upload-time = "2026-08-10T13:40:49.391Z" }, + { url = "https://files.pythonhosted.org/packages/63/2c/eb487fafc9f50ffff2b1e0b697f70fb34bf150821c08ab225aacf5583a7e/greenlet-3.5.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:1b5ed9162c0c098e0bbc2cf88a94f433c1b8926f831745252e099e5d83e17759", size = 432462, upload-time = "2026-08-10T14:30:02.309Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8b/6acf112ed8aee499f25b4d6949820fb02ac950ff9c1f3d793bd5be0599f2/greenlet-3.5.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27493374cff1d1b7919dc8126547f2aea582737e3046147b434b1e12de56389b", size = 1581342, upload-time = "2026-08-10T14:15:05.653Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/734e5f198888876b42d7616ff6644c075baf6b8a2412deadd6b0e1b8b20c/greenlet-3.5.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:12e2ee66c2aba86133f10fd99d6a8856c6d351ffb7be0e4d52ef2cc5fbb705b2", size = 1645744, upload-time = "2026-08-10T13:40:30.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/30/1f42b88dc587b5899ee50616ad56ee40cafaf225df4fb829f10183c62a5c/greenlet-3.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:49ddacd36af37735fab103846f4ee4d18a492dde72730d1699c0c8ebe30d9f18", size = 324171, upload-time = "2026-08-10T13:28:44.472Z" }, + { url = "https://files.pythonhosted.org/packages/76/e5/4dee4d8d2e603fe5fdd7b444e63219f7b9bd852c60c6214511c7157cbe88/greenlet-3.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:5f1b1ff4828cdc1aba4266aff814085d04a1d07959287219af021b838b265d52", size = 308362, upload-time = "2026-08-10T13:26:46.839Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, +] + [[package]] name = "grpc-google-iam-v1" version = "0.14.5" @@ -873,6 +950,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, ] +[[package]] +name = "playwright" +version = "1.62.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/5b/ca2abcf3aa69f9fb510215e3064f30b57fe57657c8d04ede45bb966d5606/playwright-1.62.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:d8da938f3748841a8754f2e1f0216902c1c8f8ae3720de8b32ccf8e6913a7c4f", size = 43732091, upload-time = "2026-07-31T17:00:44.178Z" }, + { url = "https://files.pythonhosted.org/packages/af/1a/0bfbe9904350961f4dbb713f04342e40d548c5fc26c8157bd13617c81492/playwright-1.62.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:db755ab27db21a04186f1fe8169888e42356086e439b1059b923ef417f0b6034", size = 42510842, upload-time = "2026-07-31T17:00:48.596Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/c0486b407ad0699a250f6bbe3066fca95344009a99ca66e88ca175c69dc1/playwright-1.62.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:5108bd5b3e87169ddf269feee097da5893af7f8aea4634dfc840518d64c1f1da", size = 43732093, upload-time = "2026-07-31T17:00:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/43/6b/b24aebc2b04bffcb342bccf96e287c78b363e1615bed5cea97500cc0393a/playwright-1.62.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ba33bae6a13b3d9d354c751cb618af357d20fe1d57767cbcce52079bbef17ad3", size = 47748926, upload-time = "2026-07-31T17:00:56.438Z" }, + { url = "https://files.pythonhosted.org/packages/36/43/b4b18bdc87e1949568fffdcde3ff9a0456266b2d0c6d4432cc34d89ea6eb/playwright-1.62.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db2d76613a57ad844362ce42f7d0c2fa26b19a4f7a46d4f76b891c631e6e5aff", size = 47441423, upload-time = "2026-07-31T17:01:00.404Z" }, + { url = "https://files.pythonhosted.org/packages/81/22/af5d926fc2c32a339eec00a443644bc40ab9db1dd2dd9017873c59773c0c/playwright-1.62.0-py3-none-win32.whl", hash = "sha256:e5614fa89355d7081457680324bb219f79f69c423c5cb6fa250e30b0d8aebf1c", size = 38164450, upload-time = "2026-07-31T17:01:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/4160c1033c07af98bf841ad079457dd78408a5ee0dd56cbfe50b8b6a1c22/playwright-1.62.0-py3-none-win_amd64.whl", hash = "sha256:92c0d98ed04eb35af557b709875edba415b1f548bdb22ddb5bb3e1e6c835c2f1", size = 38164458, upload-time = "2026-07-31T17:01:08.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" }, +] + [[package]] name = "proto-plus" version = "1.28.4" @@ -1047,6 +1143,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" @@ -1155,6 +1263,7 @@ dependencies = [ { name = "google-genai" }, { name = "lxml" }, { name = "openai" }, + { name = "playwright" }, { name = "python-dotenv" }, { name = "requests" }, ] @@ -1171,6 +1280,7 @@ requires-dist = [ { name = "google-genai", specifier = ">=1.0.0" }, { name = "lxml", specifier = ">=4.9.0" }, { name = "openai", specifier = ">=1.0.0" }, + { name = "playwright", specifier = ">=1.55.0" }, { name = "python-dotenv", specifier = "~=1.2.1" }, { name = "requests", specifier = ">=2.32.2" }, ] diff --git a/vision_aid/site_audit/crawler.py b/vision_aid/site_audit/crawler.py index 03ec374..d6e0454 100644 --- a/vision_aid/site_audit/crawler.py +++ b/vision_aid/site_audit/crawler.py @@ -4,8 +4,10 @@ import ipaddress import json +import os import re import socket +import time import xml.etree.ElementTree as ET from collections import deque from dataclasses import dataclass @@ -111,6 +113,13 @@ "replytocom", "wc-ajax", } +BOT_CHALLENGE_MARKERS = ( + "robot challenge screen", + "checking the site connection security", + "d1rozh26tys225.cloudfront.net/loader.svg", + "sg-captcha", + "sgcaptcha", +) @dataclass(frozen=True) @@ -193,6 +202,96 @@ def validate_public_url(url: str) -> str: return normalized +def _looks_like_bot_challenge(body: str, final_url: str) -> bool: + """Recognize soft-200 interstitials that must never be audited as pages.""" + path = urlparse(final_url).path.lower() + if "/.well-known/sgcaptcha" in path or "/.well-known/captcha" in path: + return True + sample = body[:250_000].lower() + return any(marker in sample for marker in BOT_CHALLENGE_MARKERS) + + +def _browser_fetch( + url: str, + *, + timeout: int, + max_bytes: int, + accepted_types: tuple[str, ...], +) -> tuple[str, str, str]: + """Solve JavaScript interstitials in Chromium while preserving SSRF checks.""" + from playwright.sync_api import sync_playwright + + original = validate_public_url(url) + executable = os.getenv("DAT_CHROMIUM_EXECUTABLE", "").strip() or None + with sync_playwright() as playwright: + browser = playwright.chromium.launch( + executable_path=executable, + headless=True, + args=["--disable-dev-shm-usage", "--no-sandbox"], + ) + try: + context = browser.new_context( + user_agent=USER_AGENT, + extra_http_headers={ + "Accept-Language": REQUEST_HEADERS["Accept-Language"] + }, + service_workers="block", + ) + page = context.new_page() + + def safe_route(route, request): + request_url = request.url + if request_url.startswith(("http://", "https://")): + try: + validate_public_url(request_url) + except ValueError: + route.abort("blockedbyclient") + return + route.continue_() + + page.route("**/*", safe_route) + response = page.goto( + original, + wait_until="domcontentloaded", + timeout=timeout * 1_000, + ) + deadline = time.monotonic() + min(25, timeout) + while _looks_like_bot_challenge(page.content(), page.url): + if time.monotonic() >= deadline: + raise RuntimeError( + "The site's browser security challenge could not be completed" + ) + page.wait_for_timeout(1_000) + + # Reload after the challenge so ``response.body()`` is the actual + # requested resource rather than the initial interstitial. + response = page.reload( + wait_until="domcontentloaded", + timeout=timeout * 1_000, + ) + final_url = validate_public_url(page.url) + if response is None: + raise RuntimeError("The browser did not return a page response") + content_type = response.headers.get("content-type", "").split(";", 1)[0].lower() + if accepted_types and not any(content_type == item for item in accepted_types): + raise ValueError(f"Unsupported content type: {content_type or 'unknown'}") + content = response.body() + if len(content) > max_bytes: + raise ValueError( + f"Response exceeds the {max_bytes // (1024 * 1024)} MB limit" + ) + encoding_match = re.search( + r"charset=([A-Za-z0-9._-]+)", response.headers.get("content-type", "") + ) + encoding = encoding_match.group(1) if encoding_match else "utf-8" + body = content.decode(encoding, errors="replace") + if _looks_like_bot_challenge(body, final_url): + raise RuntimeError("The site returned a browser security challenge") + return body, canonicalize_url(final_url), content_type + finally: + browser.close() + + def _request( session: requests.Session, url: str, @@ -262,6 +361,13 @@ def _request( body = b"".join(chunks).decode(encoding, errors="replace") final_url = canonicalize_url(current) response.close() + if _looks_like_bot_challenge(body, final_url): + return _browser_fetch( + current, + timeout=max(timeout, 45), + max_bytes=max_bytes, + accepted_types=accepted_types, + ) return body, final_url, content_type raise ValueError("Too many redirects") From c0633ad84a0c1ee0027bd0d1dfe4f0409a245ed9 Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 18:49:11 -0400 Subject: [PATCH 05/15] Add token and cost report to whole-site ZIP --- tests/test_site_audit.py | 22 ++++++++- vision_aid/site_audit/jobs.py | 8 ++++ vision_aid/site_audit/report.py | 83 +++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index cf4f877..f06c2a5 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -337,18 +337,36 @@ def test_report_zip_contains_cover_csv_and_summary(self): candidate_count=1, pages=[{"url": "https://www.abilitybazaar.com/", "status": "complete"}], page_results=[ - {"page_url": "https://www.abilitybazaar.com/", "csv_report": csv_text} + { + "page_url": "https://www.abilitybazaar.com/", + "csv_report": csv_text, + "summary": { + "total_input_tokens": 1_000, + "total_output_tokens": 500, + "estimated_cost_usd": 0.012345, + }, + } ], ) self.assertEqual(report.total_findings, 1) self.assertIn(b"Whole-Site Accessibility Audit Report", report.html_bytes) self.assertIn(b"Page Summary", report.html_bytes) self.assertIn(b"page_url", report.csv_bytes) + self.assertEqual(report.total_tokens, 1_500) + self.assertEqual(report.estimated_cost_usd, 0.012345) with zipfile.ZipFile(io.BytesIO(report.zip_bytes)) as archive: self.assertEqual( set(archive.namelist()), - {"DAT-whole-site-report.html", "DAT-findings.csv", "DAT-summary.json"}, + { + "DAT-whole-site-report.html", + "DAT-findings.csv", + "DAT-summary.json", + "DAT-token-and-cost-report.html", + }, ) + usage_report = archive.read("DAT-token-and-cost-report.html") + self.assertIn(b"1,500", usage_report) + self.assertIn(b"$0.012345", usage_report) if __name__ == "__main__": diff --git a/vision_aid/site_audit/jobs.py b/vision_aid/site_audit/jobs.py index 948c2a0..e0c5adf 100644 --- a/vision_aid/site_audit/jobs.py +++ b/vision_aid/site_audit/jobs.py @@ -455,6 +455,10 @@ def public_job(self, job: dict) -> dict: "discovery_sources", "capped", "total_findings", + "total_input_tokens", + "total_output_tokens", + "total_tokens", + "estimated_cost_usd", "report_ready", "email_sent", "email_delivery_status", @@ -782,6 +786,10 @@ def finalize(self, job_id: str) -> dict: "report_ready": True, "report_objects": objects, "total_findings": report.total_findings, + "total_input_tokens": report.total_input_tokens, + "total_output_tokens": report.total_output_tokens, + "total_tokens": report.total_tokens, + "estimated_cost_usd": report.estimated_cost_usd, "email_sent": True, "email_delivery_status": "accepted", "email_message_id": delivery["message_id"], diff --git a/vision_aid/site_audit/report.py b/vision_aid/site_audit/report.py index d3f3016..f73fb77 100644 --- a/vision_aid/site_audit/report.py +++ b/vision_aid/site_audit/report.py @@ -22,6 +22,10 @@ class SiteReport: total_findings: int pages_succeeded: int pages_failed: int + total_input_tokens: int + total_output_tokens: int + total_tokens: int + estimated_cost_usd: float def _csv_rows(csv_text: str | None, page_url: str) -> list[dict[str, str]]: @@ -154,6 +158,74 @@ def _render_csv(all_rows: list[dict[str, str]]) -> bytes: return output.getvalue().encode("utf-8-sig") +def _render_usage_report( + *, + base_url: str, + model: str, + page_results: list[dict], +) -> tuple[bytes, dict]: + """Build a standalone human-readable token and estimated-cost report.""" + usage_rows = [] + total_input = 0 + total_output = 0 + total_cost = 0.0 + for result in page_results: + summary = result.get("summary") or {} + input_tokens = int(summary.get("total_input_tokens") or 0) + output_tokens = int(summary.get("total_output_tokens") or 0) + estimated_cost = float(summary.get("estimated_cost_usd") or 0.0) + total_input += input_tokens + total_output += output_tokens + total_cost += estimated_cost + usage_rows.append( + { + "url": str(result.get("page_url") or ""), + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "estimated_cost_usd": estimated_cost, + } + ) + + rows_html = "".join( + "" + f"{html.escape(row['url'])}" + f"{row['input_tokens']:,}{row['output_tokens']:,}" + f"{row['total_tokens']:,}" + f"${row['estimated_cost_usd']:,.6f}" + for row in usage_rows + ) or 'No model usage was recorded.' + total_tokens = total_input + total_output + generated = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + document = f""" + +DAT token and cost report

Token and Estimated Cost Report

+

Site: {html.escape(base_url)}
Model: {html.escape(model)}
+Generated: {generated}

+
{total_input:,}Input tokens
+
{total_output:,}Output tokens
+
{total_tokens:,}Total tokens
+
${total_cost:,.6f}Estimated cost
+

Costs are estimates calculated by the DAT pipeline from the recorded model usage for each completed page.

+

Page-by-page usage

+ +{rows_html}
PageInput tokensOutput tokensTotal tokensEstimated cost (USD)
""" + totals = { + "total_input_tokens": total_input, + "total_output_tokens": total_output, + "total_tokens": total_tokens, + "estimated_cost_usd": round(total_cost, 8), + "pages": usage_rows, + } + return document.encode("utf-8"), totals + + def build_site_report( *, base_url: str, @@ -188,6 +260,11 @@ def build_site_report( all_rows=all_rows, ) csv_bytes = _render_csv(all_rows) + usage_html, usage = _render_usage_report( + base_url=base_url, + model=model, + page_results=page_results, + ) summary_json = json.dumps( { "base_url": base_url, @@ -198,6 +275,7 @@ def build_site_report( "candidate_count": candidate_count, "capped": capped, "total_findings": len(all_rows), + "usage": usage, "pages": [ { "url": page.get("url"), @@ -217,6 +295,7 @@ def build_site_report( archive.writestr("DAT-whole-site-report.html", html_bytes) archive.writestr("DAT-findings.csv", csv_bytes) archive.writestr("DAT-summary.json", summary_json) + archive.writestr("DAT-token-and-cost-report.html", usage_html) return SiteReport( html_bytes=html_bytes, @@ -225,4 +304,8 @@ def build_site_report( total_findings=len(all_rows), pages_succeeded=sum(1 for page in normalized_pages if page.get("status") == "complete"), pages_failed=sum(1 for page in normalized_pages if page.get("status") == "failed"), + total_input_tokens=usage["total_input_tokens"], + total_output_tokens=usage["total_output_tokens"], + total_tokens=usage["total_tokens"], + estimated_cost_usd=usage["estimated_cost_usd"], ) From 2aebfa8c7a0fa531730e21a9adf12a918b8536ea Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 18:54:00 -0400 Subject: [PATCH 06/15] Wait for anti-bot redirect navigation to settle --- vision_aid/site_audit/crawler.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vision_aid/site_audit/crawler.py b/vision_aid/site_audit/crawler.py index d6e0454..0cc05cd 100644 --- a/vision_aid/site_audit/crawler.py +++ b/vision_aid/site_audit/crawler.py @@ -219,6 +219,7 @@ def _browser_fetch( accepted_types: tuple[str, ...], ) -> tuple[str, str, str]: """Solve JavaScript interstitials in Chromium while preserving SSRF checks.""" + from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import sync_playwright original = validate_public_url(url) @@ -256,11 +257,22 @@ def safe_route(route, request): timeout=timeout * 1_000, ) deadline = time.monotonic() + min(25, timeout) - while _looks_like_bot_challenge(page.content(), page.url): + while True: if time.monotonic() >= deadline: raise RuntimeError( "The site's browser security challenge could not be completed" ) + try: + current_body = page.content() + current_url = page.url + except PlaywrightError: + # A successful interstitial may replace itself while this + # loop is sampling the DOM. Let navigation settle instead + # of treating that normal transition as a page failure. + page.wait_for_timeout(250) + continue + if not _looks_like_bot_challenge(current_body, current_url): + break page.wait_for_timeout(1_000) # Reload after the challenge so ``response.body()`` is the actual From 6f3c6641add5393ba9dcbb246ee2bebb6b59be68 Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sat, 29 Aug 2026 21:00:40 -0400 Subject: [PATCH 07/15] Ensure visible delivery for SMTP self-mail --- tests/test_site_audit.py | 46 +++++++++++++++++++++++++++++++++++ vision_aid/site_audit/jobs.py | 28 ++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index f06c2a5..04f3af5 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -211,6 +211,7 @@ def test_email_records_smtp_acceptance_and_omits_zip_by_default(self): "SMTP_PASSWORD": "secret", "SMTP_FROM": "sender@example.com", "DAT_EMAIL_ATTACH_REPORT": "false", + "DAT_SMTP_SELF_DELIVERY_FALLBACK": "", }, ), mock.patch( "vision_aid.site_audit.jobs.smtplib.SMTP", return_value=smtp_context @@ -228,6 +229,50 @@ def test_email_records_smtp_acceptance_and_omits_zip_by_default(self): self.assertEqual(receipt["recipient"], "recipient@example.com") self.assertFalse(receipt["attachment_included"]) self.assertTrue(receipt["message_id"].startswith("<")) + self.assertFalse(receipt["self_delivery_fallback_used"]) + self.assertEqual(receipt["accepted_recipient_count"], 1) + self.assertEqual( + smtp.send_message.call_args.kwargs["to_addrs"], + ["recipient@example.com"], + ) + + def test_self_delivery_adds_visible_fallback_copy(self): + smtp = mock.Mock() + smtp.send_message.return_value = {} + smtp_context = mock.Mock() + smtp_context.__enter__ = mock.Mock(return_value=smtp) + smtp_context.__exit__ = mock.Mock(return_value=False) + with mock.patch.dict( + os.environ, + { + "SMTP_HOST": "smtp.office365.com", + "SMTP_PORT": "587", + "SMTP_USER": "abilitybazaar@visionaid.org", + "SMTP_PASSWORD": "secret", + "SMTP_FROM": "abilitybazaar@visionaid.org", + "DAT_EMAIL_ATTACH_REPORT": "false", + "DAT_SMTP_SELF_DELIVERY_FALLBACK": "ram@visionaid.org", + }, + ), mock.patch( + "vision_aid.site_audit.jobs.smtplib.SMTP", return_value=smtp_context + ): + receipt = send_report_email( + recipient="abilitybazaar@visionaid.org", + base_url="https://dat.visionaid.org/", + pages=26, + findings=290, + download_url="https://dat.example.com/report", + report_zip=b"zip", + ) + message = smtp.send_message.call_args.args[0] + self.assertEqual(message["To"], "abilitybazaar@visionaid.org") + self.assertEqual(message["Cc"], "ram@visionaid.org") + self.assertEqual( + smtp.send_message.call_args.kwargs["to_addrs"], + ["abilitybazaar@visionaid.org", "ram@visionaid.org"], + ) + self.assertTrue(receipt["self_delivery_fallback_used"]) + self.assertEqual(receipt["accepted_recipient_count"], 2) def test_email_raises_when_recipient_is_refused(self): smtp = mock.Mock() @@ -242,6 +287,7 @@ def test_email_raises_when_recipient_is_refused(self): "SMTP_USER": "sender@example.com", "SMTP_PASSWORD": "secret", "SMTP_FROM": "sender@example.com", + "DAT_SMTP_SELF_DELIVERY_FALLBACK": "", }, ), mock.patch( "vision_aid.site_audit.jobs.smtplib.SMTP", return_value=smtp_context diff --git a/vision_aid/site_audit/jobs.py b/vision_aid/site_audit/jobs.py index e0c5adf..84f1d2c 100644 --- a/vision_aid/site_audit/jobs.py +++ b/vision_aid/site_audit/jobs.py @@ -161,6 +161,9 @@ def send_report_email( smtp_user = os.getenv("SMTP_USER", "").strip() smtp_password = os.getenv("SMTP_PASSWORD", "") smtp_from = os.getenv("SMTP_FROM", smtp_user).strip() + self_delivery_fallback = os.getenv( + "DAT_SMTP_SELF_DELIVERY_FALLBACK", "" + ).strip() if not all((smtp_host, smtp_user, smtp_password, smtp_from)): raise RuntimeError("SMTP delivery is not configured") @@ -172,6 +175,15 @@ def send_report_email( message["Date"] = format_datetime(_now()) message_id = make_msgid(domain=(smtp_from.rsplit("@", 1)[-1] or None)) message["Message-ID"] = message_id + envelope_recipients = [recipient] + self_delivery_fallback_used = ( + bool(self_delivery_fallback) + and recipient.strip().casefold() == smtp_from.casefold() + and self_delivery_fallback.casefold() != recipient.strip().casefold() + ) + if self_delivery_fallback_used: + message["Cc"] = self_delivery_fallback + envelope_recipients.append(self_delivery_fallback) attach_report = os.getenv("DAT_EMAIL_ATTACH_REPORT", "").strip().lower() in { "1", "true", @@ -201,7 +213,7 @@ def send_report_email( refused = smtp.send_message( message, from_addr=smtp_from, - to_addrs=[recipient], + to_addrs=envelope_recipients, ) if refused: refused_recipients = ", ".join(str(item) for item in refused) @@ -213,6 +225,8 @@ def send_report_email( "recipient": recipient, "smtp_host": smtp_host, "attachment_included": attach_report and len(report_zip) <= 18 * 1024 * 1024, + "self_delivery_fallback_used": self_delivery_fallback_used, + "accepted_recipient_count": len(envelope_recipients), } @@ -795,6 +809,12 @@ def finalize(self, job_id: str) -> dict: "email_message_id": delivery["message_id"], "email_accepted_at": delivery["accepted_at"], "email_attachment_included": delivery["attachment_included"], + "email_self_delivery_fallback_used": delivery[ + "self_delivery_fallback_used" + ], + "email_accepted_recipient_count": delivery[ + "accepted_recipient_count" + ], "completed_at": _now(), "updated_at": _now(), "last_error": firestore.DELETE_FIELD, @@ -826,6 +846,12 @@ def resend_report(self, job_id: str) -> dict: "email_message_id": delivery["message_id"], "email_accepted_at": delivery["accepted_at"], "email_attachment_included": delivery["attachment_included"], + "email_self_delivery_fallback_used": delivery[ + "self_delivery_fallback_used" + ], + "email_accepted_recipient_count": delivery[ + "accepted_recipient_count" + ], "email_resend_count": firestore.Increment(1), "updated_at": _now(), "last_error": firestore.DELETE_FIELD, From a5bfaa420fc4b35d1ae12a3049cb01bd286685aa Mon Sep 17 00:00:00 2001 From: Ramakrishna Raju Date: Sun, 30 Aug 2026 15:53:13 -0400 Subject: [PATCH 08/15] Add uploaded URL-list batch audits --- README.md | 12 ++ entry_points/api_server.py | 20 ++- index.html | 255 ++++++++++++++++++++++++------ tests/test_site_audit.py | 142 +++++++++++++++++ vision_aid/site_audit/__init__.py | 7 +- vision_aid/site_audit/jobs.py | 125 +++++++++++---- vision_aid/site_audit/report.py | 47 +++++- vision_aid/site_audit/url_list.py | 205 ++++++++++++++++++++++++ 8 files changed, 718 insertions(+), 95 deletions(-) create mode 100644 vision_aid/site_audit/url_list.py diff --git a/README.md b/README.md index 909446b..076147a 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,18 @@ Serves the UI and the audit API from one process. Users can paste their own API key into the form instead of configuring one server-side; per-request keys take priority over the environment. +The web app offers two asynchronous bulk-audit modes with the same model, API +key, email, progress, consolidated findings, and token/cost reporting: + +- **Full Site — Crawl and Email** (default): starts with one public URL, + discovers same-site pages, and audits up to 200 pages. +- **URL List — Batch and Email**: accepts a UTF-8/plain-text `.txt` file or a + Word `.docx` file containing up to 200 public HTTP(S) URLs. It deduplicates + the list and audits the supplied pages directly without crawling. + +The list upload is limited to 2 MB. Legacy `.doc` files are not supported; +save them as `.docx` or plain text first. + The audit endpoints stream **NDJSON** — progress events, one JSON object per line, then a final `{"type":"result"}` object. Parsing that body with a single `res.json()` fails; the front end branches on content type. diff --git a/entry_points/api_server.py b/entry_points/api_server.py index fe4d7d3..135cddf 100644 --- a/entry_points/api_server.py +++ b/entry_points/api_server.py @@ -42,6 +42,7 @@ from processing_scripts.llm_client.client import is_openai_model, is_gemini_model # noqa: E402 from vision_aid.site_audit.crawler import validate_public_url # noqa: E402 from vision_aid.site_audit.jobs import get_coordinator # noqa: E402 +from vision_aid.site_audit.url_list import decode_uploaded_urls # noqa: E402 # ── Multi-page splitting ───────────────────────────────────────────────────── @@ -648,10 +649,10 @@ def _handle_validate_key(self): except ValueError as exc: self._send_json({"valid": False, "error": str(exc)}, 400) - def _read_json_body(self) -> dict: + def _read_json_body(self, *, max_bytes: int = 64 * 1024) -> dict: """Read and parse one bounded JSON request body.""" length = int(self.headers.get("Content-Length", 0)) - if length <= 0 or length > 64 * 1024: + if length <= 0 or length > max_bytes: raise ValueError("A JSON request body is required") try: value = json.loads(self.rfile.read(length)) @@ -694,14 +695,25 @@ def _handle_site_audit_config(self): self._send_json({"success": True, "config": config}) def _handle_create_site_audit(self): - """Create a durable whole-site audit and return immediately.""" + """Create a durable crawl or uploaded URL-list audit and return immediately.""" try: - data = self._read_json_body() + data = self._read_json_body(max_bytes=3 * 1024 * 1024) + audit_mode = str(data.get("audit_mode") or "crawl").strip().lower() + source_file_name = "" + uploaded_urls = None + if audit_mode == "url_list": + source_file_name, uploaded_urls = decode_uploaded_urls( + data.get("url_file_name", ""), + data.get("url_file_base64", ""), + ) job = get_coordinator().create_job( base_url=data.get("url", ""), email=data.get("email", ""), model=data.get("model", ""), api_key=data.get("api_key", ""), + audit_mode=audit_mode, + uploaded_urls=uploaded_urls, + source_file_name=source_file_name, ) except ValueError as exc: self._send_json({"success": False, "error": str(exc)}, 400) diff --git a/index.html b/index.html index 07c9316..1efac68 100644 --- a/index.html +++ b/index.html @@ -301,8 +301,8 @@

Accessibility Audit Tool

- Provide an HTML file, or enter a URL to fetch and audit — with - optional recursive crawling of nested pages. + Audit one page, crawl an entire site, or upload a list of up to + 200 URLs for asynchronous batch processing and email delivery.

@@ -316,7 +316,7 @@

Accessibility Audit Tool

Input type

+
-
+ @@ -759,12 +774,32 @@

Accessibility Audit Tool

admin_authenticated: false, }; + function hasAdminAccess() { + return Boolean(ADMIN_AUDIT_MODE) || siteAuditConfig.admin_authenticated; + } + + function applyAdminUi() { + const authenticated = siteAuditConfig.admin_authenticated; + document.querySelectorAll(".protected-admin-nav").forEach((item) => { + item.hidden = !authenticated; + }); + + if (!ADMIN_AUDIT_MODE) { + document.querySelectorAll(".admin-mode-tab").forEach((item) => { + item.hidden = !authenticated; + }); + } + + document.getElementById("adminLoginBtn").hidden = authenticated; + document.getElementById("adminLogoutBtn").hidden = !authenticated; + } + function isAsyncBulkMode(mode = getAuditMode()) { return mode === "site" || mode === "url-list"; } function showSavedKeyStatus() { - if (!ADMIN_AUDIT_MODE || apiKeyInput.value.trim()) return; + if (!hasAdminAccess() || apiKeyInput.value.trim()) return; const provider = getProvider(modelSelect.value); apiKeyValidateStatus.hidden = false; if (provider !== "openai") { @@ -799,6 +834,7 @@

Accessibility Audit Tool

const result = await response.json(); if (response.ok && result.success) { siteAuditConfig = result.config; + applyAdminUi(); updateApiKeyField(); showSavedKeyStatus(); } @@ -814,8 +850,7 @@

Accessibility Audit Tool

function updateApiKeyField() { const provider = getProvider(modelSelect.value); - const usesSavedKey = - Boolean(ADMIN_AUDIT_MODE) && siteAuditConfig.admin_authenticated; + const usesSavedKey = hasAdminAccess(); if (provider === "openai") { apiKeyLabel.textContent = "OpenAI API Key"; apiKeyInput.placeholder = @@ -836,7 +871,7 @@

Accessibility Audit Tool

modelSelect.addEventListener("change", () => { updateApiKeyField(); - if (ADMIN_AUDIT_MODE && !apiKeyInput.value.trim()) { + if (hasAdminAccess() && !apiKeyInput.value.trim()) { loadSiteAuditConfig(modelSelect.value, false); } }); @@ -853,8 +888,7 @@

Accessibility Audit Tool

const provider = getProvider(model); const useSaved = !key && - Boolean(ADMIN_AUDIT_MODE) && - siteAuditConfig.admin_authenticated && + hasAdminAccess() && provider === "openai" && siteAuditConfig.api_key_configured; if (!key && !useSaved) { @@ -972,6 +1006,13 @@

Accessibility Audit Tool

applyAuditMode(getAuditMode()); loadSiteAuditConfig(); + document + .getElementById("adminLogoutBtn") + .addEventListener("click", async () => { + const response = await fetch("/api/logout", { method: "POST" }); + if (response.ok) window.location.assign("/"); + }); + // ── Drag-and-drop / file input ──────────────────────────────────────────── let htmlFileContent = null; diff --git a/styles.css b/styles.css index 80c7d1a..d8eadb2 100644 --- a/styles.css +++ b/styles.css @@ -253,6 +253,34 @@ a:hover { outline-offset: 2px; } +.admin-login-container { + margin-top: 1rem; + text-align: center; +} + +.admin-login-btn { + display: inline-block; + padding: 10px 28px; + border: 2px solid #0D7377; + border-radius: 6px; + background: transparent; + color: #0D7377; + font: 600 1rem 'DM Sans', sans-serif; + line-height: 1.5; + text-decoration: none; + cursor: pointer; +} + +.admin-login-btn:hover { + background: #0D7377; + color: #FFFFFF; +} + +.admin-login-btn:focus-visible { + outline: 2px solid #D4A017; + outline-offset: 3px; +} + @keyframes fadeIn { from { opacity: 0; diff --git a/tests/test_site_audit.py b/tests/test_site_audit.py index 232e5ea..3179e35 100644 --- a/tests/test_site_audit.py +++ b/tests/test_site_audit.py @@ -52,7 +52,22 @@ def test_obsolete_nested_crawl_option_is_not_offered(self): self.assertIn('value="url" checked', html) self.assertIn('class="audit-mode-tab admin-mode-tab" hidden', html) self.assertIn("Full Site — Crawl and Email", html) - self.assertIn('href="/analytics">Analytics', html) + self.assertIn( + '