From 551151a2c2ccfe0301dca1b22fcc8abf5d540f56 Mon Sep 17 00:00:00 2001 From: Aaron Phifer <191053702+aaronphifer@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:27:37 -0400 Subject: [PATCH 1/2] feat: prepare VariaQ 0.8.0rc1 release candidate Add controlled scientific benchmarks, paired-solver evidence and the comparison workspace while retaining schema-v1 contracts. Persist campaign attempts and recover web tasks without retrying execution. Harden report writes, local web requests and capability discovery, and fix unprivileged Windows symlink tests. Include upgrade/security documentation, Linux and Windows regression coverage, and installed base-wheel checks without optional runtime dependencies. --- .github/workflows/ci.yml | 38 +- .github/workflows/windows.yml | 2 +- .gitignore | 1 + CHANGELOG.md | 16 + README.md | 21 +- ROADMAP.md | 43 +- SECURITY.md | 25 +- benchmarks/maxcut_v0.8_unwarmed.campaign.json | 73 ++++ benchmarks/maxcut_v0.8_warmed.campaign.json | 73 ++++ docs/benchmarking-ui.md | 124 ++++++ docs/releases/v0.8.0-hardening.md | 141 ++++++ docs/scientific-benchmarks.md | 149 +++++++ docs/web-ui-architecture.md | 10 + docs/web-ui.md | 18 +- pyproject.toml | 16 +- src/variaq/__init__.py | 2 +- src/variaq/analysis/comparison_view.py | 113 +++++ src/variaq/analysis/metrics.py | 7 +- src/variaq/analysis/models.py | 2 + src/variaq/analysis/operations.py | 89 ++-- src/variaq/analysis/paired.py | 372 ++++++++++++++++ src/variaq/analysis/reports.py | 156 ++++++- src/variaq/campaigns/model.py | 7 + src/variaq/campaigns/runner.py | 58 +-- src/variaq/campaigns/store.py | 56 ++- src/variaq/capabilities.py | 37 +- src/variaq/cli.py | 36 +- src/variaq/experiments/runner.py | 122 +++++- src/variaq/experiments/storage.py | 28 +- src/variaq/outputs.py | 51 ++- src/variaq/services.py | 176 ++++++-- src/variaq/solvers/cudaq_qaoa.py | 24 +- src/variaq/solvers/qaoa_shared.py | 10 +- src/variaq/solvers/qiskit_qaoa.py | 4 + src/variaq/web/app.py | 88 +++- src/variaq/web/schemas.py | 70 ++- src/variaq/web/server.py | 6 +- src/variaq/web/static/app.css | 161 +++++-- src/variaq/web/static/app.js | 51 ++- src/variaq/web/static/app2.js | 118 +++-- src/variaq/web/static/benchmark.js | 161 +++++++ src/variaq/web/task_journal.py | 137 ++++++ src/variaq/web/tasks.py | 123 +++++- src/variaq/web/templates/base.html | 6 +- src/variaq/web/templates/campaign_detail.html | 2 +- src/variaq/web/templates/campaign_new.html | 42 +- src/variaq/web/templates/comparison.html | 43 ++ src/variaq/web/templates/dashboard.html | 18 +- src/variaq/web/templates/run_detail.html | 9 +- tests/test_campaigns.py | 29 +- tests/test_capabilities.py | 13 + tests/test_comparison_view.py | 158 +++++++ tests/test_identity.py | 2 +- tests/test_paired_benchmarks.py | 408 ++++++++++++++++++ tests/test_release_hardening.py | 165 +++++++ tests/web/conftest.py | 13 +- tests/web/test_campaigns_api.py | 21 + tests/web/test_comparison_summary.py | 41 ++ tests/web/test_comparison_workspace.py | 341 +++++++++++++++ tests/web/test_task_recovery.py | 197 +++++++++ tests/web/test_web_security.py | 74 ++++ 61 files changed, 4243 insertions(+), 354 deletions(-) create mode 100644 benchmarks/maxcut_v0.8_unwarmed.campaign.json create mode 100644 benchmarks/maxcut_v0.8_warmed.campaign.json create mode 100644 docs/benchmarking-ui.md create mode 100644 docs/releases/v0.8.0-hardening.md create mode 100644 docs/scientific-benchmarks.md create mode 100644 src/variaq/analysis/comparison_view.py create mode 100644 src/variaq/analysis/paired.py create mode 100644 src/variaq/web/static/benchmark.js create mode 100644 src/variaq/web/task_journal.py create mode 100644 src/variaq/web/templates/comparison.html create mode 100644 tests/test_comparison_view.py create mode 100644 tests/test_paired_benchmarks.py create mode 100644 tests/test_release_hardening.py create mode 100644 tests/web/test_comparison_summary.py create mode 100644 tests/web/test_comparison_workspace.py create mode 100644 tests/web/test_task_recovery.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 290387f..1a966d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,42 @@ jobs: - name: Build distributions run: python -m build + base-wheel: + name: Base wheel without optional dependencies + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install build pytest + - run: python -m build + - run: python -m pip install dist/*.whl + - name: Verify unavailable backends and missing web hint + run: | + python - <<'PY' + import importlib.util + import json + import subprocess + import sys + for name in ("numpy", "qiskit", "cudaq", "fastapi"): + assert importlib.util.find_spec(name) is None, name + result = subprocess.run([sys.executable, "-m", "variaq", "capabilities", "--json"], capture_output=True, text=True, check=True, timeout=30) + envelope = json.loads(result.stdout) + assert envelope["schema_version"] == "1" + solvers = {item["name"]: item for item in envelope["data"]["solvers"]} + assert solvers["exact"]["available"] + for name in ("qaoa", "cudaq-cpu", "cudaq-gpu"): + assert not solvers[name]["installed"], name + assert not solvers[name]["available"], name + web = subprocess.run([sys.executable, "-m", "variaq", "web"], capture_output=True, text=True, timeout=30) + assert web.returncode == 2, web + assert "variaq[web]" in web.stdout + web.stderr, web + PY + - name: Test installed base wheel + run: python -m pytest tests/test_problems.py tests/test_experiments.py tests/test_identity.py tests/test_solvers.py tests/test_json_contract.py tests/test_capabilities.py tests/test_campaigns.py tests/test_paired_benchmarks.py tests/test_comparison_view.py tests/test_release_hardening.py + test-windows: runs-on: windows-latest timeout-minutes: 15 @@ -40,7 +76,7 @@ jobs: - name: Install run: python -m pip install -e '.[dev,web]' - name: Test - run: pytest tests/test_problems.py tests/test_new_families.py tests/test_identity.py tests/test_capabilities.py tests/test_json_contract.py tests/test_cli.py tests/test_experiments.py tests/test_bqm.py tests/test_family_qaoa.py tests/test_campaigns.py tests/test_analysis.py tests/test_sense_and_historical.py tests/web + run: python -m pytest -q - name: Lint run: ruff check . - name: Formatting diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0efc17c..a849103 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -20,7 +20,7 @@ jobs: - name: Install run: python -m pip install -e '.[dev]' - name: Test core, storage, CLI JSON, capabilities, BQM, family baselines, structured output - run: pytest tests/test_problems.py tests/test_experiments.py tests/test_identity.py tests/test_cli.py tests/test_solvers.py tests/test_capabilities.py tests/test_json_contract.py tests/test_bqm.py tests/test_family_qaoa.py tests/test_new_families.py + run: pytest tests/test_problems.py tests/test_experiments.py tests/test_identity.py tests/test_cli.py tests/test_solvers.py tests/test_capabilities.py tests/test_json_contract.py tests/test_bqm.py tests/test_family_qaoa.py tests/test_new_families.py tests/test_campaigns.py tests/test_analysis.py tests/test_sense_and_historical.py tests/test_paired_benchmarks.py tests/test_comparison_view.py tests/test_release_hardening.py - name: Lint run: ruff check . - name: Formatting diff --git a/.gitignore b/.gitignore index 67bde63..cb9e87b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ data/artifacts/* # Legacy/root-level runtime paths used by older local invocations. /variaq.db /variaq.db-* +*.web-worker.lock /problems/ # Local credentials and environment files diff --git a/CHANGELOG.md b/CHANGELOG.md index b23f769..1c5b119 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ VariaQ follows semantic versioning while pre-1.0. Release dates will be added when a public release is created. +## 0.8.0rc1 — local release candidate (unreleased) + +- Added controlled scientific campaign matrices, authoritative per-instance quality, + conservative paired-solver measurements, variability, null reasons and expanded exports. +- Added the responsive comparison workspace, reviewed experiment setup and local reproduction. +- Persisted web task status in additive SQLite tables with an OS-held single worker lock. + Interrupted work retains completed attempts; duplicate web requests reconnect without retries. + Campaign attempts now persist before the next solver call, with recoverable membership provenance. +- Removed solver execution from capability discovery; reads no longer select CUDA-Q targets. +- Added first-backend-call timing with explicit overlap and lazy-initialization scope. + Cold GPU state remains unverified, and native calls cannot be safely cancelled. +- Fixed canonical stored exact references, ambiguous duplicate campaign slots, + bounded web configuration, report symlink writes and cross-origin/forged-host requests. +- Preserved experiment, campaign, report, CLI and API schema version 1 and existing history. +- See [hardening and upgrade notes](docs/releases/v0.8.0-hardening.md). + ## 0.7.0 — local web UI - Added an optional, local-first web interface (`pip install 'variaq[web]'`, diff --git a/README.md b/README.md index 4b8bdc7..d61b731 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,14 @@ measure when quantum methods are useful, competitive, or impractical. **Classical · GPU · Quantum** +## Reproducible scientific benchmarks (v0.8 candidate) + +The first v0.8 slice adds two bounded, opt-in MaxCut campaign matrices and +per-instance paired analysis via `--compare paired_solvers`. Use the existing +campaign and reporting commands; see [methodology and examples](docs/scientific-benchmarks.md). +GPU availability and missing timing data are reported explicitly. Simulator +results do not establish quantum advantage. The local candidate package is `0.8.0rc1`; no release is published. + ## Why VariaQ exists Cross-framework comparisons are easy to distort accidentally. Different @@ -34,7 +42,7 @@ physical QPU. ## Current capabilities -VariaQ 0.7 supports domain-neutral optimization problem families: +VariaQ 0.8 supports domain-neutral optimization problem families: - **MaxCut** — weighted graph partitioning into two sets. - **Assignment** — tasks/resources with scores, costs, capacities, demands, and @@ -54,7 +62,7 @@ support: depends on the installed quantum backend and the binary-variable budget (see `variaq capabilities --json`). - **Graph Partitioning**: BQM lowering is implemented and tested, but quantum - support is experimental and not advertised for 0.7.0. + support is experimental in the v0.8 candidate. Solvers: @@ -62,7 +70,7 @@ Solvers: - seeded multistart local-search heuristic, - Qiskit local statevector QAOA, - CUDA-Q `qpp-cpu` QAOA, -- CUDA-Q `nvidia` and `nvidia-fp64` adapters when compatible hardware exists. +- CUDA-Q `nvidia` adapters with explicit FP32/FP64 precision when compatible hardware exists. Cross-cutting features: @@ -287,3 +295,10 @@ failure-handling, and remote-execution safety validation where applicable. ## License VariaQ is licensed under the [Apache License 2.0](LICENSE). + +The v0.8 candidate includes a guided experiment workflow +and scientific comparison workspace. See [benchmarking UI](docs/benchmarking-ui.md) +for evidence selection, timing scopes, exports, reproduction, and current limits. + +See [candidate hardening and upgrade notes](docs/releases/v0.8.0-hardening.md) for +recovery, compatibility, security boundaries and current verification. diff --git a/ROADMAP.md b/ROADMAP.md index 0c1341b..6512225 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -83,9 +83,46 @@ These are directions, not implemented features or delivery commitments. - analysis-visualization charts driven by structured `AnalysisResult` data; - safe report downloads confined to known report artifacts. -## v0.8 and later — later exploration - -Future candidates after 0.7 (none committed): richer interactive charts on the +## v0.8 — reproducible scientific benchmarking (local candidate) + +First development slice (implemented in this checkout, not a release): + +- small opt-in MaxCut matrices over sizes, graph seeds, solver repeats, and local + classical/Qiskit/CUDA-Q backends with existing resource guards; +- authoritative per-instance quality and exact-reference gaps, feasibility, + conservative repeat pairing, source-run provenance, and repeat variability; +- complete-call wall time and available compute/setup/warmup layers, with explicit + null/reason metadata for missing measurements and unverified cold starts; +- separate numerical backend agreement from speed and sampled quality; +- additive schema-v1 JSON, CSV, and Markdown reporting, documented methodology, + and dependency-free regression coverage. + +Web interface development slice (implemented in the candidate): + +- cohesive dark-first responsive dashboard and navigation; +- guided experiment recipes, capability-aware solver selection, advanced QAOA + configuration, plan invalidation, and local task recovery messaging; +- bounded comparison workspace over authoritative Python instance/repeat data, + explicit pairing, null reasons, timing scopes, per-configuration scaling, + and selected JSON/CSV/Markdown report exports; +- canonical run measurements, configuration/environment provenance, and local + reproduction with saved lineage; +- keyboard-accessible scientific explanations and alternative chart data tables. + +Durable SQLite task snapshots and recovery of interrupted web tasks are implemented. +Duplicate web submissions reconnect to the original task, and campaign attempts are +saved before the next solver call. CLI reruns remain explicit new executions. +See [web benchmarking workflow](docs/benchmarking-ui.md) and +[candidate hardening](docs/releases/v0.8.0-hardening.md). Hard native-call cancellation +remains deferred. + +Fresh-process GPU cold-start verification, trustworthy peak memory telemetry, +confidence intervals, and interrupting hung native calls remain future design work. +See [scientific benchmark methodology](docs/scientific-benchmarks.md). + +## After v0.8 — later exploration + +Future candidates (none committed): richer interactive charts on the existing API, campaign execution progress refinement, stronger classical baselines evaluated only where they add scientific value, Graph Partition quantum research, and optional authenticated LAN mode for trusted labs. diff --git a/SECURITY.md b/SECURITY.md index 572a315..7de689b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ Ordinary non-sensitive defects may use a normal bug report. ## Current model -VariaQ 0.2 executes and stores experiments locally. Normal tests require no +VariaQ executes and stores experiments locally. Normal tests require no credentials and no network. Never commit API keys, IBM credentials, cloud tokens, private keys, or user experiment databases. @@ -26,4 +26,25 @@ queue/execution/charge timing, remain disabled in normal tests, and never be an implicit fallback. A future interface may require both `--backend ibm:` and -`--allow-qpu`. This functionality does not exist in VariaQ 0.2. +`--allow-qpu`. This functionality does not exist in VariaQ. + +## Local web boundary (v0.8 candidate) + +The default listener binds to `127.0.0.1`. Default local mode validates Host +headers and refuses cross-origin/browser cross-site writes. It has no authentication +and is not an internet service. Explicit non-loopback mode retains the trusted-lab +boundary, prints a warning, and permits arbitrary hostnames; it is not hardened for +DNS rebinding, hostile LAN clients or public exposure. Local applications can invoke +the JSON API without browser Origin headers. + +Requests validate bounded generator dimensions and solver budgets, in addition to +the existing run-count and solver state/memory guards. They provide no shell, raw SQL +or arbitrary filesystem selector. Imported problems are JSON data with canonical +family validation, not executable adapters. SQLite, problem/report directories and +the adjacent worker lock are trusted local files; do not share writable access with +untrusted users. Do not rename/delete the worker lock while the server is alive. +Report exporters refuse output symlinks, and downloads enforce known artifact names +and root containment. These checks do not defend against a hostile local process +racing filesystem operations. Native GPU calls have no safe in-process timeout or +cancellation. Stop the process if it hangs, inspect preserved records, and explicitly +plan subsequent work; no automatic retry is performed. diff --git a/benchmarks/maxcut_v0.8_unwarmed.campaign.json b/benchmarks/maxcut_v0.8_unwarmed.campaign.json new file mode 100644 index 0000000..229df6e --- /dev/null +++ b/benchmarks/maxcut_v0.8_unwarmed.campaign.json @@ -0,0 +1,73 @@ +{ + "campaign_format_version": "1", + "name": "maxcut-v0.8-unwarmed", + "family": "maxcut", + "problem_sizes": [ + 4, + 6, + 8 + ], + "problem_seeds": [ + 11, + 42 + ], + "solvers": [ + "exact", + "heuristic", + "qaoa", + "cudaq-cpu", + "cudaq-gpu" + ], + "repeats": 3, + "base_seed": 42, + "solver_config": { + "exact": { + "parameters": { + "max_states": 256 + } + }, + "heuristic": { + "parameters": { + "restarts": 4, + "max_passes": 16 + } + }, + "qaoa": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": false + } + }, + "cudaq-cpu": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": false, + "precision": "fp64", + "max_statevector_bytes": 1048576 + } + }, + "cudaq-gpu": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": false, + "precision": "fp64", + "max_statevector_bytes": 1048576 + } + } + }, + "generator_parameters": { + "edge_probability": 0.4 + }, + "tags": [ + "scientific-benchmark", + "v0.8", + "unwarmed" + ], + "notes": "Opt-in bounded simulator benchmark. No quantum advantage claim. Unwarmed does not prove fresh GPU state." +} diff --git a/benchmarks/maxcut_v0.8_warmed.campaign.json b/benchmarks/maxcut_v0.8_warmed.campaign.json new file mode 100644 index 0000000..b93bfb9 --- /dev/null +++ b/benchmarks/maxcut_v0.8_warmed.campaign.json @@ -0,0 +1,73 @@ +{ + "campaign_format_version": "1", + "name": "maxcut-v0.8-warmed", + "family": "maxcut", + "problem_sizes": [ + 4, + 6, + 8 + ], + "problem_seeds": [ + 11, + 42 + ], + "solvers": [ + "exact", + "heuristic", + "qaoa", + "cudaq-cpu", + "cudaq-gpu" + ], + "repeats": 3, + "base_seed": 42, + "solver_config": { + "exact": { + "parameters": { + "max_states": 256 + } + }, + "heuristic": { + "parameters": { + "restarts": 4, + "max_passes": 16 + } + }, + "qaoa": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": true + } + }, + "cudaq-cpu": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": true, + "precision": "fp64", + "max_statevector_bytes": 1048576 + } + }, + "cudaq-gpu": { + "parameters": { + "p": 1, + "optimizer_trials": 8, + "shots": 128, + "warmup": true, + "precision": "fp64", + "max_statevector_bytes": 1048576 + } + } + }, + "generator_parameters": { + "edge_probability": 0.4 + }, + "tags": [ + "scientific-benchmark", + "v0.8", + "warmed" + ], + "notes": "Opt-in bounded simulator benchmark. No quantum advantage claim. Unwarmed does not prove fresh GPU state." +} diff --git a/docs/benchmarking-ui.md b/docs/benchmarking-ui.md new file mode 100644 index 0000000..c925661 --- /dev/null +++ b/docs/benchmarking-ui.md @@ -0,0 +1,124 @@ +# Scientific benchmarking in the local web UI + +This is the unreleased `0.8.0rc1` candidate; +CLI, API, campaign, report, and SQLite versions remain compatible with schema 1. +Start the existing loopback web service with the web extra installed: + +```sh +variaq --db data/variaq.sqlite3 web +``` + +The dashboard offers **New Experiment** and **Compare Results**. “Fully recorded +campaigns” counts campaign memberships meeting the definition's requested run +count; it includes failed attempts and reruns. This is not durable execution-state +or successful-solve accounting. Recent activity and capability availability are +separate: a historic GPU result can exist when today's GPU is unavailable. +All quantum backends currently shown are local simulators; no physical QPU or +quantum advantage is established. + +## Plan and run + +Choose a problem family and a deterministic generated reference recipe, then +select compatible, currently available solvers. Problem seeds define mathematical +instances; solver seeds control search and sampling. Classical exact and heuristic +solvers are the initial defaults. The small recipe uses size 6 and problem seed 11; +the scaling recipe uses sizes 4, 6, 8 and problem seeds 11, 42. Canonical problem IDs +are recorded on execution. This workflow generates reference instances; it does not +import an arbitrary filesystem path or directly select an existing problem file. + +Advanced controls expose generator parameters and shared QAOA depth, candidates, +shots, explicit warmup and CUDA-Q GPU precision (CPU remains FP64). The existing campaign solver overrides +carry these values into the runner. Review the execution plan's work counts, +availability, exact enumeration, and estimated simulator resources before starting. +Any setup edit invalidates the reviewed plan. A plan is not a guarantee that an +optional backend remains available when execution starts. + +One in-process execution slot is shared by campaigns and reproductions. Recorded +progress counts persisted attempts, including unsuccessful outcomes; skipped work +can leave that count below the plan. Refreshing the experiment form can reconnect +to the last browser-tracked task. SQLite snapshots survive server restarts and +unfinished tasks become `interrupted`, with saved run counts and an explanation. +An OS-held worker lock prevents a second web worker from executing on the same +database. Duplicate web submissions reconnect to the original task, including +terminal tasks, and never retry automatically. Historical campaign definitions +without a journal are refused for execution because their state is uncertain. +Inspect preserved evidence before explicitly creating a new campaign with a +different name. Explicit CLI reruns still append a new execution; they cannot +resume an interrupted native operation. + +## Compare evidence + +Choose a campaign or up to 500 explicit run IDs. The comparison API analyzes at +most 500 matching records in a window, with visible total, offset and next/previous +controls. Solver, backend and instance filters apply in the database. Size and +warmup filters apply within that labelled window. Large campaign windows can split +repeat pairs; focus an instance to inspect a smaller compatible selection. + +Objective and gap charts display one explicitly named instance. The comparison +summary shows its selected exact optimum, valid quality-run count, optimum-achievement +rate and mean measurable gap. Achievement counts only successful feasible evaluated +runs with a consistent reference (including selected exact runs), using the existing +quality tolerance of 1e-9. Its numerator/denominator is visible. Undefined relative +gaps have a separate denominator; a nonzero absolute gap to a zero optimum has +no percentage. Measured zero gaps show a labelled baseline dot and explicit counts; +unavailable gaps remain absent and labelled unavailable. + +Quality and timing charts show Python descriptive means per solver/backend, +configuration, precision, warmup mode, benchmark execution, environment and timing +scope. Successful infeasible or unevaluable runs do not enter these summaries. +Chart focus and evidence-instance filtering are separate controls. Changing chart +focus does not silently narrow the other filters or exported selection. Quality, +gap and timing charts show at most 12 configurations, with an explicit notice when +further configurations +remain in the complete summary tables. Comparison configurations use C labels; +existing scaling series use separate S labels. +Source IDs, full controls and individual measurements remain available; grouping +does not establish matched experimental controls or change recorded pairing. +Seeds/candidate angles can vary across repeats under the same configured budget. +Long methodological explanations are available under “How to read these comparisons.” +Only feasible objectives enter the quality chart. References come from consistent feasible exact +runs in selected evidence. Filtering out exact runs can make gaps unavailable; +that is reported with a reason, not converted to zero. Wall, compute and explicit +warmup charts retain nulls and use seconds. Failed attempt duration is separate +from successful performance. Historical timing scopes can remain unknown. + +Scaling retains existing Python repeat timing means (successful timing observations +can include infeasible solutions); it shows them separately for each problem +instance, solver, backend, configuration and warmup mode. Different instances remain distinct points, +with identity in tooltips and accessible data tables. No quality is pooled across +instances and no speedup ratio is inferred. Repeat tables expose count, mean, +median, sample standard deviation and range; fewer than two observations have no +standard deviation. These are not confidence intervals. + +A recorded pair shares problem identity, benchmark execution and repeat slot. +Failed paired records are labelled; absent or duplicate slots remain incomplete or +ambiguous. Standalone runs remain unpaired. Candidate-expectation agreement is +reported only when Python's existing controlled-pair rules allow it. + +The complete selected window (not just the currently displayed instance) exports +through the existing report layer to JSON, CSV and Markdown, with source IDs, +measurements, pairing, variability and missing reasons. A report of the same +selected evidence retains its artifact generation timestamp when regenerated. +Known filenames and report-root confinement govern downloads and comparison +exports; planted output symlinks are refused. No filesystem or shell selector is +provided to the browser. + +## Inspect and reproduce + +Run detail shows authoritative re-evaluation against up to 500 explicitly selected +runs of the same problem, including exact reference source IDs and missing reasons. +Original reported measurements remain labelled separately. Expand the canonical +problem, solver configuration, environment, timing scopes, warnings and lineage. +Eligible local reproductions use the existing runner and append a linked new run; +wall times and sampled outcomes are not expected to be identical. + +Charts have data tables, numbered observations or distinct point shapes, explicit +units, readable legends and keyboard-accessible explanations. Narrow comparison +tables become labelled cards; optional historical data tables scroll within their +containers. Tooltip help can be dismissed with Escape. + +Remaining measurement work includes verified GPU cold starts, trustworthy +peak-memory telemetry, confidence intervals, native-call cancellation and manual +assistive-technology review. Durable interrupted-task recovery is implemented; +see [candidate hardening](releases/v0.8.0-hardening.md). No hosting, remote worker, +scientific schema migration or published release is part of this candidate. diff --git a/docs/releases/v0.8.0-hardening.md b/docs/releases/v0.8.0-hardening.md new file mode 100644 index 0000000..4040986 --- /dev/null +++ b/docs/releases/v0.8.0-hardening.md @@ -0,0 +1,141 @@ +# VariaQ v0.8 candidate hardening and upgrade notes + +The local candidate identifies itself as `0.8.0rc1`. No release has been published. +The recommendation is **CONDITIONAL GO for local candidate evaluation**. Linux +validation passed. Windows CI must pass before claiming Windows release readiness. +The separate BB plugin has verified Linux compatibility with this candidate on +BB 0.46 / SDK 0.6.37; its Windows execution remains unverified. + +## Installation and upgrade + +Use Python 3.12 or newer and an isolated environment. Build the current checkout +with `python -m build`, then install the generated wheel: + +```sh +python -m venv .venv +# Activate the environment using your platform's usual command. +python -m pip install 'dist/variaq-0.8.0rc1-py3-none-any.whl[web]' +variaq --version +variaq web +``` + +The base package has no required third-party runtime dependencies. `[web]` adds +the local interface; `[quantum]` adds Qiskit 2.x; `[cudaq]` requires CUDA-Q 0.16.x +and a compatible platform/runtime. Install simulator extras only where supported. +Capability discovery describes imports, target discovery and device discovery; +it does not prove a solver invocation works. Driver installation is outside VariaQ. + +Stop the old server before upgrading. Back up SQLite, problem definitions and +reports together while no writer is active. Scientific schema version remains 1, +and historical record JSON is not rewritten. Starting the web app adds a +`web_tasks` table and an adjacent `.web-worker.lock` file. Do not rename or delete +that lock while a server is alive. A generated historical fixture created with the +committed v0.7 store was read successfully with its record JSON unchanged; this +does not establish compatibility with every possible user database. + +## Execution and recovery + +Campaign attempts are persisted individually before the next solver starts. +Campaign IDs and web task IDs are recorded before saving a result. On restart, +the owner of the released OS worker lock reconciles orphan campaign memberships +and recorded task counts, then marks unfinished tasks `interrupted`. + +Refreshing the browser reconnects to the saved task. Submitting the same campaign +definition again returns its existing task, including a terminal task, without +rerunning it. Repeating a reproduction request for the same source run also returns +its original task. Historical campaigns without a task journal cannot be resumed +through the web UI, because their execution state is uncertain. Inspect preserved +records before explicitly planning a new campaign with a different name. CLI +campaign reruns remain an explicit append operation. + +Task failure or interruption differs from solver results. Recorded attempts have +`success`, `failed` or `unavailable` status; unsupported family/solver combinations +are counted as `skipped` without inventing records. “Completed runs” counts saved +attempts, including failures and unavailable backends. It does not mean every +attempt succeeded. Matrix or persistence failures terminate the task instead of +fabricating successful progress. + +An in-flight solver result may be lost when the process exits. Native GPU calls +have no safe in-process cancellation or execution deadline. The optional +`nvidia-smi` metadata probe has a three-second timeout; that is not a solver timeout. +If a native call hangs, stop the process and inspect preserved records. No automatic +retry occurs. The lock coordinates web workers using the same database; it does +not serialize independently launched CLI jobs or unrelated native-library users. + +## Scientific interpretation + +Objectives are evaluated from the canonical problem and solution. A successful, +feasible exact solution can establish a reference; inconsistent exact references +produce an unknown optimum. Missing reference gaps and missing measurements remain +null, with explanations. Historical metadata does not imply current precision, +timing scope, paired identity or verified cold state. + +Paired evidence requires compatible problem/configuration, repeat, seed, parameter +digest, precision and warmup controls. Reproduction preserves lineage and inputs, +but does not guarantee identical hardware timing or stochastic outcomes across +package, device or runtime versions. See [scientific methodology](../scientific-benchmarks.md). + +Wall time covers the solver call and excludes persistence. Backend construction, +explicit warmup, expectation calls, parameter search and final sampling have +separate documented scopes. First-backend-evaluation time captures lazy work on +the first expectation call, and overlaps either warmup or expectation evaluation. +Parameter search also contains expectation evaluation. Do not sum overlapping +measurements. Construction timing is not proof of GPU initialization time. + +Bounded local validation on Linux/Python 3.12 used Qiskit 2.5.2 and CUDA-Q +0.16.0.post1 on an RTX 4060. Forty attempts succeeded across exact, heuristic, +Qiskit, CUDA-Q CPU and CUDA-Q GPU, covering unwarmed and explicit-warmup modes. +Twenty attempts used a nontrivial four-node MaxCut instance with optimum 4; twelve +paired backend expectation observations agreed within `3.11e-15`. Twenty additional +attempts exercised a zero-optimum instance. These small ideal-simulator checks +establish bounded numerical agreement, not performance scaling, GPU speedup or +physical quantum advantage. Fresh processes do not establish a cold device or +empty driver/kernel caches; `cold_start_verified` remains null. + +## Security and compatibility boundaries + +The default listener is `127.0.0.1`. Local mode validates Host headers and rejects +cross-site/cross-origin browser writes. Explicit non-loopback binding warns and +retains trusted-lab behavior without authentication. Do not expose it as a public +service. Request dimension/budget guards complement run-count and state/memory +limits; the explicit run-count override can still authorize expensive work. + +Imported problems are validated JSON data, not executable plugins. Reports reject +unsafe filenames and existing output symlinks; downloads enforce artifact names +and root containment. Writable local storage is trusted. These checks do not stop +a hostile local process racing filesystem operations. See [security policy](../../SECURITY.md). + +The tested dependency set passes `pip check`. Qiskit and CUDA-Q have bounded major +or minor ranges, while several web dependencies have minimum-only constraints. +The current test results do not guarantee compatibility with future dependency +majors. No credentials, paid services, physical QPU jobs or driver changes were +used in this hardening pass. + +The separate `bb-plugin-variaq` candidate accepts stable VariaQ 0.6.x and exact +`0.8.0rc1`, preserving schema 1. Its existing campaign analysis/report surfaces +forward `paired_solvers` evidence and expanded CSV artifacts. It targets BB 0.46 +and SDK 0.6.37. Actual local tests passed against this checkout (123 tests) and +the immutable VariaQ v0.6.0 tag (22 integration tests), with bounded Qiskit and +CUDA-Q CPU/GPU candidate coverage. Type/SDK/build/package checks and live BB CLI +workflows passed. One existing SDK scanner check remains skipped for false +positives; plugin Windows execution and six development dependency advisories +remain documented limits. The production dependency audit reports zero advisories. +This is separately verified plugin compatibility, not a published plugin release. + +## Validation and remaining work + +The full local suite, product Ruff scope, source/wheel builds, fresh wheel installs, +historical fixture, forced-exit recovery and desktop/mobile browser flows were +verified during this pass. Browser testing included report download, reproduction, +refresh/restart recovery and accessibility scans at three viewport sizes. Whole-tree +Ruff still reports unrelated untracked research files; those files were preserved +and excluded from the source distribution. + +Before a public release, run the configured Windows CI, including file-lock and +recovery tests, including the report symlink privilege portability regression. +Windows CUDA-Q and Windows plugin execution were not verified. Defer device cold-start isolation, safe +native termination, authenticated network hosting and a larger telemetry system. +The comparison view's bounded input window can split very large paired groups; +use explicit bounded selections rather than assuming it summarizes an unlimited +history. No automatic recovery execution, distributed queue or remote worker was +introduced. diff --git a/docs/scientific-benchmarks.md b/docs/scientific-benchmarks.md new file mode 100644 index 0000000..e9cb921 --- /dev/null +++ b/docs/scientific-benchmarks.md @@ -0,0 +1,149 @@ +# Scientific benchmarks: first v0.8 slice + +The opt-in MaxCut suite uses the existing campaign, experiment history, analysis, +and reporting layers. It requires no optional packages for classical runs or for +the new core tests. All quantum backends in this suite are local simulators. +Numerical agreement and simulator timing do not demonstrate quantum advantage. + +## Bounded suite + +`benchmarks/maxcut_v0.8_unwarmed.campaign.json` and +`benchmarks/maxcut_v0.8_warmed.campaign.json` each request 90 runs: +3 sizes (4, 6, 8 nodes), 2 graph seeds (11, 42), 3 solver repeats, and 5 solvers +(exact, heuristic, Qiskit, CUDA-Q CPU, CUDA-Q GPU). They use edge probability 0.4, +solver seeds 42–44, depth 1, 8 identical candidates per matched repeat, 128 shots, +and FP64 for both CUDA-Q targets. Exact enumeration is guarded at 256 states; +heuristics use 4 restarts and 16 passes; CUDA-Q has a 1 MiB statevector guard. +Existing quantum variable/memory guards also apply. Execution is sequential. + +These are **campaign files**, not legacy `variaq suite` instance files. The old +suite remains supported. Use `campaign plan` before running. A missing optional +package or GPU produces a durable `unavailable` run with its reason. A solver +exception produces a failed run; neither is scored as a feasible solution or as +successful performance. Skipped unsupported solvers have no experiment record. + +```bash +variaq --db data/v08.sqlite3 campaign plan benchmarks/maxcut_v0.8_unwarmed.campaign.json --json +variaq --db data/v08.sqlite3 campaign run benchmarks/maxcut_v0.8_unwarmed.campaign.json --json +variaq --db data/v08.sqlite3 campaign run benchmarks/maxcut_v0.8_warmed.campaign.json --json +# Substitute the campaign_id returned by each run: +variaq --db data/v08.sqlite3 analyze campaign campaign-ID --compare paired_solvers \ + --include-failed --include-unavailable --json +variaq --db data/v08.sqlite3 report campaign campaign-ID --compare paired_solvers \ + --output-dir reports/v08 +``` + +On Linux, prefix each campaign run with `timeout 120s` for an external hard wall +limit. Algorithm budgets bound work, but the synchronous runner cannot interrupt +a hung native backend call. Campaigns save each completed solver attempt before starting the next. +Interrupted execution retains completed attempts; an in-flight native call's +result may not have been saved. There is no automatic retry or native cancellation. +Do not run this optional quantum matrix in ordinary CI. For a core-only trial, +copy a campaign definition and retain only `exact`/`heuristic` and their overrides. + +Identical existing problem files are checked and reused. Re-running a definition +appends new run IDs to the same deterministic campaign ID; benchmark IDs separate +execution batches. Use explicit run selection to analyze only one execution. + +## Measurements and pairing + +`--compare paired_solvers` adds `instances` to `AnalysisResult` and a comparison +through the existing `ComparisonSummary` model. It works independently of +`group_by`. Each instance contains the canonical problem, exact-reference run IDs, +per-run measurements, and repeat summaries. Original source records remain intact. + +- Solutions are re-evaluated by the original problem. Reported solver objectives + are retained separately, with an agreement flag. +- Optimality gaps require consistent feasible exact runs in the selected data. + Best observed heuristic/sampled results never become proven optima. Absolute + gaps respect minimize/maximize semantics. Percentage gaps divide by the absolute + reference; equality gives zero, including a zero optimum (the existing + convention). A nonzero gap with zero reference has null percentage and a reason. +- Pairing requires the same mathematical problem, benchmark ID, and repeat index. + Historical records use configuration seed within the benchmark. Duplicate slots + are ambiguous and remain unpaired. Standalone runs receive measurements but + are not paired. Missing slots cannot distinguish skipped, filtered, or interrupted + runs; this limitation is explicit rather than inferred. +- Numerical comparison requires identical QAOA candidates, budgets, warmup flags, + and seeds, plus complete candidate expectations. It reports maximum expectation + delta separately from sampled solution quality and durations. Precision is + retained for interpretation; no universal tolerance or speedup is asserted. +- Wall time measures the complete solver call, including validation, lowering, + setup/imports, and warmup; database writes and environment capture are excluded. + Solver compute time uses each solver's reported scope. For quantum solvers this + covers warmup, expectation evaluation, and sampling. Initialization, search, + expectation, warmup, and sampling layers are retained when measured. Search and + expectation times overlap and must not be added together. +- Unavailable measurements are null with `missing_measurements` reasons. Failed + and unavailable attempt wall time is retained separately. Missing timing layers + never become zero. An unrequested warmup has no measurement in paired analysis. +- Repeat summaries stay within an instance, solver/backend, requested configuration, + and warmup mode. They include observation counts, mean, median, sample standard + deviation, minimum, maximum, source IDs, and status counts. Standard deviation is + null with fewer than two observations. Different seeds vary sampling/search; + exact and heuristic results can remain deterministic. + +`unwarmed` means no explicit warmup, so first-use costs remain in timing. +`explicit_warmup` performs the existing first-candidate warmup before search; +its cost remains inside wall/compute time and is reported separately. +Capability discovery imports dependencies and queries targets/devices without +executing a solver or selecting a CUDA-Q target. Imports/device discovery can still +initialize runtime state. `first_backend_evaluation_seconds` measures the first +expectation call, including lazy compilation/setup. It overlaps warmup (when +requested) or expectation evaluation (when unwarmed), so it must never be added +to those layers. CUDA-Q initialization separately covers imports, target selection +and kernel construction; it does not claim to isolate all first-use GPU costs. +The solver invocation index helps identify first use within the runner, but shared +process/backend/device caches prevent a verified GPU cold-start claim. +`cold_start_verified` is null with explanatory metadata. Separate CLI invocations +reduce process reuse but do not prove fresh GPU/driver state. + +`compare quantum --json` now compares expectations within matched repeats instead +of comparing distinct repeat candidate sequences. With no complete backend pair, +its delta is null with a reason. Existing summary comparisons remain available; +use paired analysis for scientific per-repeat conclusions. + +## Reporting and compatibility + +The existing JSON and Markdown artifacts now include instance measurements, +pair observations, variability, and missing-data explanations. CSV exports add +`_runs.csv`, `_pairs.csv`, and `_repeats.csv`, retaining nested scalar fields and +encoding lists as JSON. Empty paired analyses emit header-only CSV files. Empty +CSV cells represent null; reasons are exported alongside measurements. + +Ordering is deterministic. Report IDs now include analysis contents, so different +comparison/scaling requests cannot collide. A report freezes its generation time +across exports; a separately regenerated artifact can have a new generation time. +Run IDs, actual durations, environment versions, and timestamps naturally differ +between new benchmark executions. Reproducibility means controlled inputs and +traceable observations, not identical elapsed times or cross-backend samples. + +CLI schema, campaign format, report format, and SQLite schema stay at version 1. +No migration or historical record rewrite is required. Missing older metadata is +handled conservatively. Percentage gaps without proven optima and incomplete +numerical comparisons now return null; consumers must handle these values. +The local API accepts `paired_solvers` and safely serves the new CSV filenames. +No remote execution, IBM hardware, credentials, or hosted services are added. + +GPU precision now uses `nvidia` with an explicit precision option, following +[NVIDIA's target documentation](https://nvidia.github.io/cuda-quantum/latest/using/backends/sims/svsims.html). +This avoids the deprecated `nvidia-fp64` alias's native stdout warning, which would +otherwise contaminate CLI JSON. Historical backend names stay in their source records. + +## Audit and follow-up + +Reused: canonical problem IDs/evaluators, exact and heuristic solvers, generic BQM +lowering, matched candidate generation/digests, backend capability discovery, +quantum memory guards, layered timers, append-only experiment history, campaign +planning/execution/membership, statistical summaries, and JSON/CSV/Markdown exports. +Existing tests already cover solver limits, Hamiltonian/bit ordering, cross-backend +equivalence, structured CLI output, historical data, and web security. + +This slice adds deterministic campaign fixtures, reliable pairing and reference +quality, missing-data semantics, provenance, repeat statistics, and regression +coverage. The local web comparison workspace now exposes these analysis/API fields: +instance selection, timing scopes, warmup modes, missing reasons, quality versus +numerical agreement, and paired variability. See [web benchmarking workflow](benchmarking-ui.md). Fresh-process cold +start measurement, peak memory telemetry, confidence intervals, and hard native +backend cancellation remain deferred. Durable task interruption recovery is implemented +as described in [candidate hardening](releases/v0.8.0-hardening.md). diff --git a/docs/web-ui-architecture.md b/docs/web-ui-architecture.md index 5d01668..59b3dfa 100644 --- a/docs/web-ui-architecture.md +++ b/docs/web-ui-architecture.md @@ -69,3 +69,13 @@ HTMX dependency for the handful of dynamic regions; charts need real JS anyway. - Campaign task tracking is intentionally in-process and single-flight; a richer progress model can be added behind the same `/api/v1/campaigns/tasks/` endpoints without breaking the UI. + +## v0.8 recovery update + +Execution remains sequential in one server thread. An adjacent OS-held file lock +allows only one web worker per database, and additive SQLite task snapshots retain +request identity across restarts. Each campaign attempt is persisted before the +next solver starts. Recovery reconciles committed records with campaign membership +and marks unfinished tasks interrupted; it never resumes or retries them. +Browser refresh reconnects to the stored task. Native calls cannot be safely +cancelled in process. See [candidate recovery notes](releases/v0.8.0-hardening.md). diff --git a/docs/web-ui.md b/docs/web-ui.md index be26b7c..a6fcd7a 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -47,7 +47,7 @@ known VariaQ artifacts beneath the configured root. ## Default bind address and security The server binds to **127.0.0.1** by default: it is only reachable from this -machine. The 0.7 UI is local-first software with **no hardened +machine. The UI is local-first software with **no hardened internet-facing authentication or authorization**. If you deliberately bind to another address for a trusted lab LAN: @@ -94,14 +94,16 @@ plots). Reports preserve `source_run_ids`. Downloads are restricted to files that belong to a known report artifact inside the reports root — the UI cannot read arbitrary files from disk. -## Limitations (0.7.0) +## Limits and recovery (0.8 candidate) -- One campaign executes at a time per server process (deliberate, no worker - queue). Task state is in-memory: if the server is stopped mid-campaign, - already-recorded runs remain in the database, but the task progress entry is - lost. +- One web execution runs at a time per database, enforced by an OS-held worker + lock, with durable SQLite task snapshots and no worker queue. On restart, + unfinished tasks become interrupted and completed attempts remain accessible. + Duplicate web submissions reconnect without executing again. There is no automatic + retry or safe native GPU cancellation. See [recovery and upgrade details](releases/v0.8.0-hardening.md). - No accounts, collaboration, or remote execution. -- Charts cover campaign scaling analysis only. +- Comparison charts cover selected per-instance quality, gaps, timing and repeat scaling. + Missing measurements remain null, and GPU cold state is not verified. ## Security model and non-goals @@ -109,7 +111,7 @@ The web layer provides **no** shell execution, arbitrary Python import, raw SQL, arbitrary filesystem browsing, credential management, or telemetry. Write actions (campaign run, report generation) use explicit bounded schemas. -Out of scope for 0.7: physical QPU access, IBM Runtime, AWS Braket, D-Wave, +Out of scope for v0.8: physical QPU access, IBM Runtime, AWS Braket, D-Wave, cloud accounts, internet-hosted SaaS, and any domain-specific integration. ## API diff --git a/pyproject.toml b/pyproject.toml index cf29264..86089b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "variaq" -version = "0.7.0" +version = "0.8.0rc1" description = "Local-first reproducible comparison of classical, GPU-accelerated, and quantum methods" readme = "README.md" requires-python = ">=3.12" @@ -59,6 +59,20 @@ variaq = "variaq.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/variaq"] +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", + "/docs", + "/benchmarks", + "/examples", + "/*.md", + "/LICENSE", + "/pyproject.toml", + "/.github", + "/.gitignore", +] + [tool.pytest.ini_options] addopts = "-q" testpaths = ["tests"] diff --git a/src/variaq/__init__.py b/src/variaq/__init__.py index 9d131ec..7a5c1e1 100644 --- a/src/variaq/__init__.py +++ b/src/variaq/__init__.py @@ -1,6 +1,6 @@ """VariaQ: reproducible optimization experiment comparisons.""" -__version__ = "0.7.0" +__version__ = "0.8.0rc1" from variaq import adapter, lower, models from variaq.problems import ( diff --git a/src/variaq/analysis/comparison_view.py b/src/variaq/analysis/comparison_view.py new file mode 100644 index 0000000..19c7e7e --- /dev/null +++ b/src/variaq/analysis/comparison_view.py @@ -0,0 +1,113 @@ +"""Bounded UI summaries of authoritative per-instance measurements. + +This is a derived view, not a replacement for analysis or recorded pairing. +No reference, objective, gap, timing scope or run identity is reinterpreted. +""" + +from __future__ import annotations + +import json +import math +from collections import defaultdict +from typing import Any + +from variaq.analysis.metrics import make_repeat_summary +from variaq.analysis.paired import TIMING_FIELDS + + +def _finite(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _summary(runs: list[dict[str, Any]], optimum: float | None) -> dict[str, Any]: + valid = [ + r + for r in runs + if r["status"] == "success" and r["feasible"] is True and _finite(r["objective"]) + ] + comparable = [r for r in valid if _finite(r["optimality_gap"])] if _finite(optimum) else [] + achieved = [r for r in comparable if abs(r["objective"] - optimum) <= 1e-9] + gaps = [r["optimality_gap_percent"] for r in comparable if _finite(r["optimality_gap_percent"])] + zero = sum(gap == 0 for gap in gaps) + return { + "valid_run_count": len(valid), + "valid_run_ids": [r["run_id"] for r in valid], + "excluded_run_count": len(runs) - len(valid), + "reference_comparable_count": len(comparable), + "reference_comparable_run_ids": [r["run_id"] for r in comparable], + "optimum_achieved_count": len(achieved), + "optimum_achievement_rate": len(achieved) / len(comparable) if comparable else None, + "objective": make_repeat_summary([r["objective"] for r in valid]).to_dict(), + "gap_percent": make_repeat_summary(gaps).to_dict(), + "gap_counts": { + "zero": zero, + "nonzero": len(gaps) - zero, + "unavailable": len(runs) - len(gaps), + }, + "gap_state": "unavailable" if not gaps else "all_zero" if zero == len(gaps) else "nonzero", + "timing": { + field: make_repeat_summary([r["timing"][field] for r in valid]).to_dict() + for field in TIMING_FIELDS + }, + } + + +def comparison_view(instances: tuple[dict[str, Any], ...]) -> list[dict[str, Any]]: + """Keep configurations, executions, precision and timing scopes separate. + + Seeds/candidate angles vary across repeated searches under the same budget; + their exact values remain in the source measurements. Grouping never implies + matched controls or pairing. Achievement uses existing quality's 1e-9 rule. + """ + result = [] + for instance in instances: + buckets: dict[str, list[dict[str, Any]]] = defaultdict(list) + identities = {} + for run in instance["runs"]: + backend = { + k: v + for k, v in run["backend"].items() + if k not in {"metrics", "reproducibility_notes"} + } + identity = { + "solver": run["solver"], + "solver_version": run["solver_version"], + "backend": backend, + "parameters": { + k: v for k, v in run["parameters"].items() if k != "candidate_parameters" + }, + "precision": run["precision"], + "warmup_mode": run["warmup_mode"], + "timing_scopes": run["timing_scopes"], + "benchmark_id": run["benchmark_id"], + "environment": { + k: v for k, v in run["environment"].items() if k != "benchmark_protocol" + }, + "hardware": { + k: run["backend"]["metrics"][k] + for k in ("gpus", "gpu_count", "target_name", "target_type", "target_options") + if k in run["backend"]["metrics"] + }, + } + key = json.dumps(identity, sort_keys=True) + identities[key] = identity + buckets[key].append(run) + optimum = instance["reference_optimum"] + groups = [ + { + "group_id": f"config-{index + 1}", + **identities[key], + "run_ids": [r["run_id"] for r in runs], + **_summary(runs, optimum), + } + for index, (key, runs) in enumerate(sorted(buckets.items())) + ] + result.append( + { + "problem_id": instance["problem_id"], + "reference_optimum": optimum, + **_summary(instance["runs"], optimum), + "groups": groups, + } + ) + return result diff --git a/src/variaq/analysis/metrics.py b/src/variaq/analysis/metrics.py index 3ffd924..511a087 100644 --- a/src/variaq/analysis/metrics.py +++ b/src/variaq/analysis/metrics.py @@ -54,10 +54,11 @@ def best_known_objective(runs: Sequence[ExperimentRun]) -> tuple[float | None, s for r in runs if r.result.solver_name == "exact" and r.result.status is SolveStatus.SUCCESS + and r.result.feasible and r.result.objective is not None ] if exact: - return max(exact), "exact_optimum" + return _best(_dominant_sense(runs), exact), "exact_optimum" feasible = [ r.result.objective for r in runs @@ -101,9 +102,9 @@ def absolute_gap(best: float, objective: float, sense: OptimizationSense) -> flo return max(0.0, objective - best) -def relative_gap(best: float, objective: float) -> float: +def relative_gap(best: float, objective: float) -> float | None: if best == 0.0: - return 0.0 + return 0.0 if objective == best else None return abs(best - objective) / abs(best) * 100.0 diff --git a/src/variaq/analysis/models.py b/src/variaq/analysis/models.py index 9439435..b51c883 100644 --- a/src/variaq/analysis/models.py +++ b/src/variaq/analysis/models.py @@ -236,6 +236,7 @@ class AnalysisResult: comparisons: tuple[ComparisonSummary, ...] warnings: tuple[str, ...] source_run_ids: tuple[str, ...] + instances: tuple[dict[str, Any], ...] = () def to_dict(self) -> dict[str, Any]: return { @@ -245,4 +246,5 @@ def to_dict(self) -> dict[str, Any]: "comparisons": [c.to_dict() for c in self.comparisons], "warnings": list(self.warnings), "source_run_ids": list(self.source_run_ids), + "instances": list(self.instances), } diff --git a/src/variaq/analysis/operations.py b/src/variaq/analysis/operations.py index c610cda..a8b1741 100644 --- a/src/variaq/analysis/operations.py +++ b/src/variaq/analysis/operations.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import defaultdict +from statistics import median from typing import Any from variaq.analysis.metrics import ( @@ -10,7 +11,6 @@ _dominant_sense, _worst, approximation_ratio, - best_known_objective, extract_field, make_repeat_summary, relative_gap, @@ -26,6 +26,7 @@ ScalingPoint, TimingSummary, ) +from variaq.analysis.paired import _evaluation, paired_analysis from variaq.models import ExperimentRun, OptimizationSense, SolveStatus @@ -54,10 +55,8 @@ def _group_key(run: ExperimentRun, group_by: tuple[str, ...]) -> dict[str, Any]: return key -def _make_quality(runs: list[ExperimentRun]) -> QualitySummary: - successful = [ - r for r in runs if r.result.status is SolveStatus.SUCCESS and r.result.objective is not None - ] +def _make_quality(runs: list[ExperimentRun], reference_runs: list[ExperimentRun]) -> QualitySummary: + successful = [r for r in runs if _evaluation(r)[0] is not None and _evaluation(r)[1]] count = len(successful) if count == 0: return QualitySummary( @@ -70,39 +69,43 @@ def _make_quality(runs: list[ExperimentRun]) -> QualitySummary: success_at_optimum_rate=None, approximation_ratio=None, ) - objectives = [float(r.result.objective) for r in successful] + objectives = [float(_evaluation(r)[0]) for r in successful] sense = _dominant_sense(successful) - best, source = best_known_objective(successful) - if best is None: - best = _best(sense, objectives) - gaps = [relative_gap(float(best), obj) for obj in objectives] - successes = [ - 1 - for r in successful - if r.result.best_known_objective is not None - and r.result.objective is not None - and abs(float(r.result.objective) - float(r.result.best_known_objective)) <= 1e-9 - ] + references: dict[str, set[float]] = defaultdict(set) + for run in reference_runs: + obj, feasible, _ = _evaluation(run) + if run.result.solver_name == "exact" and feasible and obj is not None: + references[run.result.problem_id].add(obj) + gaps: list[float] = [] + successes: list[bool] = [] ratios: list[float] = [] - for obj in objectives: - ratio = approximation_ratio(best, obj, sense) - if ratio is not None and best is not None: - ratios.append(float(ratio)) + for run, obj in zip(successful, objectives, strict=True): + refs = references[run.result.problem_id] + if len(refs) != 1: + continue + best = next(iter(refs)) + gap = relative_gap(best, obj) + if gap is not None: + gaps.append(gap) + successes.append(abs(obj - best) <= 1e-9) + ratio = approximation_ratio(best, obj, _dominant_sense([run])) + if ratio is not None: + ratios.append(ratio) return QualitySummary( count=count, best_objective=_best(sense, objectives), worst_objective=_worst(sense, objectives), mean_objective=sum(objectives) / len(objectives), - median_objective=sorted(objectives)[len(objectives) // 2] if objectives else None, + median_objective=median(objectives), mean_gap_percent=sum(gaps) / len(gaps) if gaps else None, - success_at_optimum_rate=sum(successes) / len(successful) if successful else None, + success_at_optimum_rate=sum(successes) / len(successes) if successes else None, approximation_ratio=sum(ratios) / len(ratios) if ratios else None, ) def _make_feasibility(runs: list[ExperimentRun]) -> FeasibilitySummary: - feasible_runs = [r for r in runs if r.result.feasible] - infeasible_runs = [r for r in runs if not r.result.feasible] + feasible_runs = [r for r in runs if _evaluation(r)[1] is True] + infeasible_runs = [r for r in runs if _evaluation(r)[1] is False] feasible_sample_count: int | None = None infeasible_sample_count: int | None = None rates: list[float] = [] @@ -123,15 +126,16 @@ def _make_feasibility(runs: list[ExperimentRun]) -> FeasibilitySummary: rates.append(rate) if int(fsc) == 0: zero_feasible += 1 - if r.result.feasible and r.result.objective is not None: + objective, feasible, _ = _evaluation(r) + if feasible and objective is not None: if best_feasible is None: - best_feasible = float(r.result.objective) + best_feasible = objective else: sense = _dominant_sense([r]) if sense is OptimizationSense.MAXIMIZE: - best_feasible = max(best_feasible, float(r.result.objective)) + best_feasible = max(best_feasible, objective) else: - best_feasible = min(best_feasible, float(r.result.objective)) + best_feasible = min(best_feasible, objective) bie = r.result.backend.metrics.get("best_infeasible_energy") if bie is not None: if best_infeasible_energy is None or float(bie) < float(best_infeasible_energy): @@ -143,7 +147,7 @@ def _make_feasibility(runs: list[ExperimentRun]) -> FeasibilitySummary: feasible_sample_count=feasible_sample_count, infeasible_sample_count=infeasible_sample_count, mean_feasible_rate=sum(rates) / len(rates) if rates else None, - median_feasible_rate=sorted(rates)[len(rates) // 2] if rates else None, + median_feasible_rate=median(rates) if rates else None, min_feasible_rate=min(rates) if rates else None, max_feasible_rate=max(rates) if rates else None, zero_feasible_runs=zero_feasible, @@ -153,6 +157,7 @@ def _make_feasibility(runs: list[ExperimentRun]) -> FeasibilitySummary: def _make_timing(runs: list[ExperimentRun]) -> TimingSummary: + runs = [r for r in runs if r.result.status is SolveStatus.SUCCESS] wall = make_repeat_summary([r.result.wall_time_seconds for r in runs]) solver = make_repeat_summary([r.result.solver_time_seconds for r in runs]) init = make_repeat_summary( @@ -167,7 +172,13 @@ def _make_timing(runs: list[ExperimentRun]) -> TimingSummary: search = make_repeat_summary( [r.result.backend.metrics.get("parameter_search_seconds") for r in runs] ) - warmup = make_repeat_summary([r.result.backend.metrics.get("warmup_seconds") for r in runs]) + warmup = make_repeat_summary( + [ + r.result.backend.metrics.get("warmup_seconds") + for r in runs + if r.solver_config.parameters.get("warmup") is True + ] + ) return TimingSummary( count=len(runs), total_wall_time_seconds=wall, @@ -201,7 +212,9 @@ def _make_resource(runs: list[ExperimentRun]) -> ResourceSummary: ) -def _make_group(runs: list[ExperimentRun], key: dict[str, Any]) -> GroupSummary: +def _make_group( + runs: list[ExperimentRun], key: dict[str, Any], reference_runs: list[ExperimentRun] +) -> GroupSummary: env: dict[str, set[str]] = defaultdict(set) for r in runs: env["variaq"].add(str(r.environment.get("packages", {}).get("variaq"))) @@ -215,7 +228,7 @@ def _make_group(runs: list[ExperimentRun], key: dict[str, Any]) -> GroupSummary: count=len(runs), run_ids=tuple(r.run_id for r in runs), problem_ids=tuple(sorted({r.result.problem_id for r in runs})), - quality=_make_quality(runs), + quality=_make_quality(runs, reference_runs), feasibility=_make_feasibility(runs), timing=_make_timing(runs), resource=_make_resource(runs), @@ -356,7 +369,7 @@ def analyze_runs( scaling_x_metric: str | None = None, comparisons: tuple[str, ...] = (), ) -> AnalysisResult: - applicable = _applicable_runs(runs, query) + applicable = sorted(_applicable_runs(runs, query), key=lambda r: r.run_id) source_run_ids = tuple(r.run_id for r in applicable) warnings: list[str] = [] @@ -368,7 +381,7 @@ def analyze_runs( key = _group_key(run, query.group_by) grouped[tuple(sorted(key.items()))].append(run) - groups = [_make_group(runs, dict(key)) for key, runs in grouped.items()] + groups = [_make_group(group_runs, dict(key), applicable) for key, group_runs in grouped.items()] groups.sort(key=lambda g: str(g.group_key)) scaling_points: list[ScalingPoint] = [] @@ -376,6 +389,11 @@ def analyze_runs( scaling_points = _build_scaling_points(groups, scaling_x_metric) comparison_results: list[ComparisonSummary] = [] + instances: tuple[dict[str, Any], ...] = () + if "paired_solvers" in comparisons: + instances, paired = paired_analysis(applicable) + comparison_results.append(paired) + warnings.extend(paired.warnings) if "classical_vs_quantum" in comparisons: comparison_results.append(_classical_vs_quantum(groups)) if "qiskit_vs_cudaq" in comparisons: @@ -393,4 +411,5 @@ def analyze_runs( comparisons=tuple(comparison_results), warnings=tuple(warnings), source_run_ids=source_run_ids, + instances=instances, ) diff --git a/src/variaq/analysis/paired.py b/src/variaq/analysis/paired.py new file mode 100644 index 0000000..d197fb7 --- /dev/null +++ b/src/variaq/analysis/paired.py @@ -0,0 +1,372 @@ +"""Per-instance measurements and conservative pairing of persisted solver runs. + +No solver is executed here. Solutions are evaluated by their original problem; +only feasible exact results establish an optimum. Timing and numerical agreement +are separate observations, never evidence of quantum advantage. +""" + +from __future__ import annotations + +import json +import math +from collections import defaultdict +from itertools import combinations +from typing import Any + +from variaq.analysis.metrics import absolute_gap, make_repeat_summary +from variaq.analysis.models import ComparisonSummary +from variaq.errors import VariaQError +from variaq.models import ExperimentRun, SolveStatus +from variaq.problems.base import problem_from_dict +from variaq.solvers.qaoa_shared import QAOA_SOLVER_NAMES, candidate_parameter_digest + +TIMING_FIELDS = { + "wall_time_seconds": None, + "solver_time_seconds": None, + "initialization_seconds": "backend_initialization_seconds", + "warmup_seconds": "warmup_seconds", + "expectation_evaluation_seconds": "expectation_evaluation_seconds", + "parameter_search_seconds": "parameter_search_seconds", + "sampling_seconds": "final_sampling_seconds", + "first_backend_evaluation_seconds": "first_backend_evaluation_seconds", +} + + +def _number(value: Any) -> float | None: + if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value): + return float(value) + return None + + +def _evaluation(run: ExperimentRun) -> tuple[float | None, bool | None, str | None]: + if run.result.status is not SolveStatus.SUCCESS or run.result.solution is None: + return None, None, "No successful solution to evaluate" + try: + problem = problem_from_dict(run.problem) + if problem.problem_id != run.result.problem_id: + return None, None, "Result and embedded problem IDs differ" + evaluation = problem.evaluate(run.result.solution) + return _number(evaluation.objective), evaluation.feasible, None + except (VariaQError, ValueError, KeyError, TypeError) as exc: + return None, None, f"Problem evaluation unavailable: {exc}" + + +def _measurement(run: ExperimentRun, optimum: float | None) -> dict[str, Any]: + problem = problem_from_dict(run.problem) + objective, feasible, reason = _evaluation(run) + missing: dict[str, str] = {} + if reason: + missing.update(objective=reason, feasible=reason) + metrics = run.result.backend.metrics + timing: dict[str, float | None] = {} + for field, metric in TIMING_FIELDS.items(): + raw = metrics.get(metric) if metric else getattr(run.result, field) + value = _number(raw) + if value is not None and value < 0: + value = None + # Failed/unavailable attempt durations are not successful solver performance. + if run.result.status is not SolveStatus.SUCCESS: + value = None + timing[field] = value + if value is None: + missing[field] = ( + "No successful solve; attempt wall time is reported separately" + if run.result.status is not SolveStatus.SUCCESS + else "Not measured by this solver or absent from historical metadata" + ) + if ( + run.result.solver_name in QAOA_SOLVER_NAMES + and run.solver_config.parameters.get("warmup") is False + ): + timing["warmup_seconds"] = None + missing["warmup_seconds"] = "Warmup was not requested" + gap = ( + absolute_gap(optimum, objective, problem.sense) + if optimum is not None and objective is not None and feasible + else None + ) + percent = ( + gap / abs(optimum) * 100 if gap is not None and optimum else (0.0 if gap == 0 else None) + ) + if gap is None: + missing["optimality_gap"] = "No proven reference optimum or no feasible solution" + if percent is None: + missing["optimality_gap_percent"] = ( + "Relative gap undefined for a zero optimum" + if optimum == 0 + else missing["optimality_gap"] + ) + warmup = run.solver_config.parameters.get("warmup") + if run.result.solver_name not in QAOA_SOLVER_NAMES: + mode = "not_applicable" + elif warmup is True: + mode = "explicit_warmup" + elif warmup is False: + mode = "unwarmed" + else: + mode = "unknown" + protocol = run.environment.get("benchmark_protocol", {}) + precision = ( + metrics.get("floating_point_precision", run.result.parameters.get("precision")) + if run.result.status is SolveStatus.SUCCESS + else None + ) + if precision is None: + missing["precision"] = "Backend precision was not reported by a successful solve" + return { + "run_id": run.run_id, + "benchmark_id": run.benchmark_id, + "rerun_of": run.rerun_of, + "solver": run.result.solver_name, + "solver_version": run.result.solver_version, + "status": run.result.status.value, + "problem_id": run.result.problem_id, + "sense": problem.sense.value, + "objective": objective, + "reported_objective": run.result.objective, + "objective_matches_reported": ( + math.isclose(objective, run.result.objective, rel_tol=1e-9, abs_tol=1e-9) + if objective is not None and _number(run.result.objective) is not None + else None + ), + "feasible": feasible, + "reference_optimum": optimum, + "optimality_gap": gap, + "optimality_gap_percent": percent, + "timing": timing, + "attempt_wall_time_seconds": _number(run.result.wall_time_seconds), + "timing_scopes": { + "wall": metrics.get("wall_time_scope"), + "solver": metrics.get("solver_time_scope"), + "initialization": metrics.get("backend_initialization_scope"), + "first_backend_evaluation": metrics.get("first_backend_evaluation_scope"), + "first_backend_evaluation_overlaps": metrics.get("first_backend_evaluation_overlaps"), + }, + "warmup_mode": mode, + "cold_start_verified": protocol.get("cold_start_verified"), + "cold_start_reason": protocol.get( + "cold_start_reason", "No process/device isolation recorded" + ), + "repeat_index": protocol.get("repeat_index"), + "solver_invocation_index": protocol.get("solver_invocation_index"), + "seed": run.solver_config.seed, + "parameters": run.solver_config.parameters, + "backend": run.result.backend.to_dict(), + "precision": precision, + "requested_precision": run.solver_config.parameters.get("precision"), + "backend_available": metrics.get( + "capability_available", (True if run.result.status is SolveStatus.SUCCESS else None) + ), + "environment": run.environment, + "missing_measurements": missing, + "warnings": list(run.result.warnings), + "errors": list(run.result.errors), + } + + +def _numerical_comparison( + left: ExperimentRun | None, right: ExperimentRun | None +) -> dict[str, Any]: + result: dict[str, Any] = { + "matched_controls": None, + "max_expectation_delta": None, + "reason": "Two successful QAOA runs are required", + } + if left is None or right is None: + return result + if any(r.result.status is not SolveStatus.SUCCESS for r in (left, right)): + return result + if any(r.result.solver_name not in QAOA_SOLVER_NAMES for r in (left, right)): + result["reason"] = "Candidate expectations apply only to QAOA backend comparisons" + return result + controls = ("p", "shots", "optimizer_trials", "warmup") + configs = [r.solver_config for r in (left, right)] + candidates = [c.parameters.get("candidate_parameters") for c in configs] + if any(not c for c in candidates) or any( + c.parameters.get(field) is None for c in configs for field in controls + ): + result["reason"] = "Incomplete QAOA controls/candidate metadata" + return result + digests = [candidate_parameter_digest(c) for c in candidates] + matched = ( + digests[0] == digests[1] + and configs[0].seed == configs[1].seed + and all(configs[0].parameters[k] == configs[1].parameters[k] for k in controls) + ) + result["matched_controls"] = matched + if not matched: + result["reason"] = "QAOA candidates, budgets, seeds or warmup controls differ" + return result + values = [r.result.backend.metrics.get("candidate_expectations") for r in (left, right)] + if any(not v or len(v) != len(candidates[0]) for v in values): + result["reason"] = "Incomplete candidate expectations" + return result + if any(_number(v) is None for vs in values for v in vs): + result["reason"] = "Non-finite candidate expectations" + return result + result.update( + max_expectation_delta=max(abs(a - b) for a, b in zip(*values, strict=True)), reason=None + ) + return result + + +def paired_analysis( + runs: list[ExperimentRun], +) -> tuple[tuple[dict[str, Any], ...], ComparisonSummary]: + """Pair within one benchmark execution/repeat; never zip unrelated histories. + + Old records fall back to configuration seed only within a benchmark ID. + Duplicate solver slots are ambiguous and remain unpaired. Standalone runs + still receive measurements, but cannot establish experimental pairing. + """ + by_problem: dict[str, list[ExperimentRun]] = defaultdict(list) + for run in runs: + by_problem[run.result.problem_id].append(run) + instances: list[dict[str, Any]] = [] + pairs: list[dict[str, Any]] = [] + warnings: list[str] = [] + for problem_id, instance_runs in sorted(by_problem.items()): + instance_runs.sort(key=lambda r: r.run_id) + problem = problem_from_dict(instance_runs[0].problem) + if any( + problem_from_dict(r.problem).identity_payload() != problem.identity_payload() + for r in instance_runs + ): + warnings.append(f"{problem_id}: conflicting problem snapshots; pairing refused") + continue + exact = [ + obj + for r in instance_runs + for obj, feasible, _reason in [_evaluation(r)] + if r.result.solver_name == "exact" and feasible and obj is not None + ] + optimum = exact[0] if exact and len(set(exact)) == 1 else None + if exact and optimum is None: + warnings.append(f"{problem_id}: exact references disagree; optimum unavailable") + measurements = {r.run_id: _measurement(r, optimum) for r in instance_runs} + repeats: dict[str, list[dict[str, Any]]] = defaultdict(list) + slots: dict[tuple[str, str], dict[str, list[ExperimentRun]]] = defaultdict( + lambda: defaultdict(list) + ) + expected: dict[str, set[str]] = defaultdict(set) + for r in instance_runs: + m = measurements[r.run_id] + params = {k: v for k, v in m["parameters"].items() if k != "candidate_parameters"} + key = json.dumps( + [m["solver"], m["backend"]["name"], params, m["warmup_mode"]], sort_keys=True + ) + repeats[key].append(m) + if r.benchmark_id is None: + warnings.append(f"{r.run_id}: no benchmark ID; run is not paired") + continue + repeat = m["repeat_index"] + label = f"repeat:{repeat}" if repeat is not None else f"seed:{r.solver_config.seed}" + slots[(r.benchmark_id, label)][r.result.solver_name].append(r) + expected[r.benchmark_id].add(r.result.solver_name) + repeat_rows = [] + for key, ms in sorted(repeats.items()): + solver, backend, params, mode = json.loads(key) + repeat_rows.append( + { + "solver": solver, + "backend": backend, + "parameters": params, + "warmup_mode": mode, + "run_ids": [m["run_id"] for m in ms], + "status_counts": { + s: sum(m["status"] == s for m in ms) + for s in ("success", "failed", "unavailable") + }, + "objective": make_repeat_summary( + [m["objective"] for m in ms if m["feasible"]] + ).to_dict(), + "timing": { + f: make_repeat_summary([m["timing"][f] for m in ms]).to_dict() + for f in TIMING_FIELDS + }, + } + ) + instances.append( + { + "problem_id": problem_id, + "problem": problem.to_dict(), + "sense": problem.sense.value, + "reference_optimum": optimum, + "reference_run_ids": [ + r.run_id + for r in instance_runs + if r.result.solver_name == "exact" and _evaluation(r)[1] + ], + "reference_reason": None + if optimum is not None + else "No consistent feasible exact run in selection", + "runs": list(measurements.values()), + "repeats": repeat_rows, + } + ) + for (benchmark_id, label), solvers in sorted(slots.items()): + for a, b in combinations(sorted(expected[benchmark_id]), 2): + left = solvers[a][0] if len(solvers[a]) == 1 else None + right = solvers[b][0] if len(solvers[b]) == 1 else None + lm = measurements[left.run_id] if left else None + rm = measurements[right.run_id] if right else None + missing = { + name: "No recorded run (skipped, filtered or incomplete execution)" + if not solvers[name] + else "Ambiguous duplicate solver/repeat slot" + for name, r in ((a, left), (b, right)) + if r is None + } + numerical = _numerical_comparison(left, right) + quality_delta = None + if lm and rm and lm["feasible"] and rm["feasible"]: + quality_delta = lm["objective"] - rm["objective"] + if problem.sense.value == "minimize": + quality_delta = -quality_delta + # Raw durations are observations. No speed ratio is inferred, and + # numerical agreement is independently reported above. + pairs.append( + { + "problem_id": problem_id, + "benchmark_id": benchmark_id, + "repeat_key": label, + "left_solver": a, + "right_solver": b, + "left_run_id": left.run_id if left else None, + "right_run_id": right.run_id if right else None, + "objective_improvement_left": quality_delta, + "numerical_correctness": numerical, + "missing_pairs": missing, + "left": { + k: lm[k] + for k in ( + "status", + "objective", + "feasible", + "timing", + "warmup_mode", + "seed", + "precision", + ) + } + if lm + else None, + "right": { + k: rm[k] + for k in ( + "status", + "objective", + "feasible", + "timing", + "warmup_mode", + "seed", + "precision", + ) + } + if rm + else None, + } + ) + return tuple(instances), ComparisonSummary( + "paired_solvers", pairs, tuple(sorted(set(warnings))) + ) diff --git a/src/variaq/analysis/reports.py b/src/variaq/analysis/reports.py index c171784..caa185f 100644 --- a/src/variaq/analysis/reports.py +++ b/src/variaq/analysis/reports.py @@ -9,6 +9,7 @@ from variaq import __version__ from variaq.analysis.models import AnalysisResult +from variaq.errors import ValidationError from variaq.models import utc_now REPORT_FORMAT_VERSION = "1" @@ -21,13 +22,14 @@ def __init__(self, report_id: str, campaign_id: str | None, analysis: AnalysisRe self.report_id = report_id self.campaign_id = campaign_id self.analysis = analysis + self.generated_at = utc_now() def to_dict(self) -> dict[str, Any]: return { "report_format_version": REPORT_FORMAT_VERSION, "report_id": self.report_id, "campaign_id": self.campaign_id, - "generated_at": utc_now(), + "generated_at": self.generated_at, "variaq_version": __version__, "source_run_ids": list(self.analysis.source_run_ids), "analysis": self.analysis.to_dict(), @@ -36,6 +38,14 @@ def to_dict(self) -> dict[str, Any]: def _safe_path(directory: Path, filename: str, overwrite: bool = False) -> Path: target = directory / filename + if ( + Path(filename).name != filename + or "/" in filename + or "\\" in filename + or target.is_symlink() + or target.resolve().parent != directory.resolve() + ): + raise ValidationError("Requested file is outside the report root or is a symlink") if target.exists() and not overwrite: raise FileExistsError(f"Report file exists: {target}; use --overwrite to replace") target.parent.mkdir(parents=True, exist_ok=True) @@ -67,6 +77,28 @@ def _group_rows(groups: list[Any]) -> list[dict[str, Any]]: return rows +def _flat_row(value: dict[str, Any], prefix: str = "") -> dict[str, Any]: + """Keep nested scalar metrics useful in CSV; retain lists as lossless JSON.""" + row: dict[str, Any] = {} + for key, item in value.items(): + name = f"{prefix}{key}" + if isinstance(item, dict) and item: + row.update(_flat_row(item, name + ".")) + elif isinstance(item, (dict, list, tuple)): + row[name] = json.dumps(item, sort_keys=True) + else: + row[name] = item + return row + + +def _write_rows(path: Path, rows: list[dict[str, Any]], empty_fields: list[str]) -> None: + fields = sorted({key for row in rows for key in row}) or empty_fields + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + + def write_csv_report( report: ReportArtifact, directory: Path, *, overwrite: bool = False ) -> dict[str, Path]: @@ -100,6 +132,27 @@ def write_csv_report( writer.writerows(srows) result["scaling"] = scaling_path + if any(c["comparison_type"] == "paired_solvers" for c in analysis["comparisons"]): + run_rows = [_flat_row(r) for i in analysis["instances"] for r in i["runs"]] + pair_rows = [ + _flat_row(p) + for c in analysis["comparisons"] + if c["comparison_type"] == "paired_solvers" + for p in c["pairs"] + ] + repeat_rows = [ + _flat_row({"problem_id": i["problem_id"], **r}) + for i in analysis["instances"] + for r in i["repeats"] + ] + for name, records, fields in ( + ("runs", run_rows, ["run_id", "problem_id", "objective"]), + ("pairs", pair_rows, ["problem_id", "left_run_id", "right_run_id"]), + ("repeats", repeat_rows, ["problem_id", "solver", "warmup_mode"]), + ): + target = _safe_path(directory, f"{report.report_id}_{name}.csv", overwrite) + _write_rows(target, records, fields) + result[name] = target return result @@ -113,7 +166,7 @@ def write_markdown_report( "", f"- **VariaQ version:** {__version__}", f"- **Report format version:** {REPORT_FORMAT_VERSION}", - f"- **Generated at:** {utc_now()}", + f"- **Generated at:** {report.generated_at}", ] if report.campaign_id: lines.append(f"- **Campaign ID:** {report.campaign_id}") @@ -159,6 +212,104 @@ def write_markdown_report( ) lines.append("") + if analysis["instances"]: + lines.extend( + [ + "## Paired per-instance analysis", + "", + "Objectives are re-evaluated by the original problem. Gaps require an exact", + "reference in the selected runs. Null means unavailable; CSV/JSON carry reasons.", + "Unwarmed runs do not establish a fresh GPU context. Simulator timings do not", + "demonstrate quantum advantage. Solver compute scopes may differ from wall time.", + "", + "| problem | run | solver | status | objective | gap % | feasible | wall s | " + "compute s | init s | warmup s | mode |", + "|" + "|".join(["---"] * 12) + "|", + ] + ) + missing_notes: list[str] = [] + for instance in analysis["instances"]: + for r in instance["runs"]: + t = r["timing"] + values = [ + instance["problem_id"], + r["run_id"], + r["solver"], + r["status"], + r["objective"], + r["optimality_gap_percent"], + r["feasible"], + t["wall_time_seconds"], + t["solver_time_seconds"], + t["initialization_seconds"], + t["warmup_seconds"], + r["warmup_mode"], + ] + lines.append( + "| " + " | ".join("null" if v is None else str(v) for v in values) + " |" + ) + if r["missing_measurements"]: + reasons = "; ".join(f"{k}: {v}" for k, v in r["missing_measurements"].items()) + missing_notes.append(f"- `{r['run_id']}`: {reasons}") + lines.extend(["", "Missing measurement reasons:", "", *missing_notes]) + lines.extend( + [ + "", + "## Repeat variability", + "", + "Sample standard deviation is null with fewer than two observations.", + "", + "| problem | solver | mode | objective n | objective mean | objective std | " + "wall n | wall mean s | wall std s |", + "|" + "|".join(["---"] * 9) + "|", + ] + ) + for instance in analysis["instances"]: + for r in instance["repeats"]: + o, t = r["objective"], r["timing"]["wall_time_seconds"] + values = [ + instance["problem_id"], + r["solver"], + r["warmup_mode"], + o["count"], + o["mean"], + o["std"], + t["count"], + t["mean"], + t["std"], + ] + lines.append( + "| " + " | ".join("null" if v is None else str(v) for v in values) + " |" + ) + lines.append("") + for comparison in analysis["comparisons"]: + if comparison["comparison_type"] != "paired_solvers": + continue + lines.extend( + [ + "## Paired solver observations", + "", + "Positive objective improvement favors the left solver for either objective sense.", + "", + "| problem | repeat | left run | right run | objective improvement left | " + "max expectation delta | reason |", + "|" + "|".join(["---"] * 7) + "|", + ] + ) + for p in comparison["pairs"]: + n = p["numerical_correctness"] + values = [ + p["problem_id"], + p["repeat_key"], + p["left_run_id"], + p["right_run_id"], + p["objective_improvement_left"], + n["max_expectation_delta"], + n["reason"] or p["missing_pairs"] or "matched expectations", + ] + lines.append("| " + " | ".join("null" if v is None else str(v) for v in values) + " |") + lines.append("") + if analysis["warnings"]: lines.append("## Warnings") lines.append("") @@ -222,6 +373,7 @@ def generate_report( "campaign_id": campaign_id, "query": analysis.query.to_dict(), "source_run_ids": sorted(analysis.source_run_ids), + "analysis": analysis.to_dict(), } import hashlib diff --git a/src/variaq/campaigns/model.py b/src/variaq/campaigns/model.py index f89e060..76cbbe0 100644 --- a/src/variaq/campaigns/model.py +++ b/src/variaq/campaigns/model.py @@ -76,6 +76,13 @@ def __post_init__(self) -> None: raise ValidationError("Campaign problem_seeds cannot be empty") if not self.solvers: raise ValidationError("Campaign solvers cannot be empty") + for label, values in ( + ("problem_sizes", self.problem_sizes), + ("problem_seeds", self.problem_seeds), + ("solvers", self.solvers), + ): + if len(set(values)) != len(values): + raise ValidationError(f"Campaign {label} must not contain duplicates") if self.repeats < 1: raise ValidationError("Campaign repeats must be positive") if any(size < 1 for size in self.problem_sizes): diff --git a/src/variaq/campaigns/runner.py b/src/variaq/campaigns/runner.py index 3a3f068..649e358 100644 --- a/src/variaq/campaigns/runner.py +++ b/src/variaq/campaigns/runner.py @@ -10,7 +10,7 @@ from variaq.experiments.runner import ExperimentRunner from variaq.models import ExperimentRun, SolverConfig from variaq.problems.assignment import AssignmentProblem -from variaq.problems.base import ProblemInstance, save_problem +from variaq.problems.base import ProblemInstance, load_problem, save_problem from variaq.problems.graph_partition import GraphPartitionProblem from variaq.problems.maxcut import MaxCutProblem from variaq.problems.subset_selection import SubsetSelectionProblem @@ -81,6 +81,7 @@ def run( override_max_runs: bool = False, on_run_recorded: Any | None = None, should_abort: Any | None = None, + web_task_id: str | None = None, ) -> dict[str, Any]: """Execute a campaign sequentially. @@ -125,7 +126,12 @@ def run( campaign.family, size, seed, campaign.generator_parameters ) problem = generated_problems[key] - save_problem(problem, self.problems_dir / f"{problem.problem_id}.json") + problem_path = self.problems_dir / f"{problem.problem_id}.json" + if problem_path.exists(): + if load_problem(problem_path).identity_payload() != problem.identity_payload(): + raise ValidationError(f"Existing problem artifact differs: {problem_path}") + else: + save_problem(problem, problem_path) configs: dict[str, SolverConfig] = {} for solver_name, solver in solver_instances.items(): @@ -142,40 +148,37 @@ def run( configs[solver_name] = SolverConfig(seed=base_seed, parameters=parameters) if not configs: - summary_statuses["skipped"] += campaign.repeats * len(solver_instances) continue - try: - runs = self.runner.benchmark( - problem, - list(solver_instances.values()), - configs, - repeats=campaign.repeats, - ) - except Exception: - # If the whole benchmark matrix fails, record one failure per requested run - # and continue to the next problem instance. - for _solver_name in configs: - for _ in range(campaign.repeats): - summary_statuses["failed"] += 1 - # No run ID exists; store association is impossible. - continue - - for run in runs: + def recorded(run: ExperimentRun) -> None: all_runs.append(run) status = run.result.status.value - summary_statuses[status] = summary_statuses.get(status, 0) + 1 + summary_statuses[status] += 1 self.store.record_run( - campaign_id=campaign_id, - run_id=run.run_id, - status=status, - solver_name=run.result.solver_name, - problem_id=run.result.problem_id, - created_at=run.created_at, + campaign_id, + run.run_id, + status, + run.result.solver_name, + run.result.problem_id, + run.created_at, ) if on_run_recorded is not None: on_run_recorded(run, len(all_runs)) + self.runner.benchmark( + problem, + [solver_instances[name] for name in configs], + configs, + repeats=campaign.repeats, + on_persisted=recorded, + campaign_id=campaign_id, + web_task_id=web_task_id, + should_abort=should_abort, + ) + if should_abort is not None and should_abort(): + aborted = True + break + if aborted: summary_statuses["aborted"] = 1 return { @@ -199,4 +202,5 @@ def run( "status_summary": summary_statuses, "problem_ids": sorted({run.result.problem_id for run in all_runs}), "run_ids": [run.run_id for run in all_runs], + "skipped_runs": summary_statuses["skipped"], } diff --git a/src/variaq/campaigns/store.py b/src/variaq/campaigns/store.py index 1f31f43..23ef552 100644 --- a/src/variaq/campaigns/store.py +++ b/src/variaq/campaigns/store.py @@ -72,7 +72,12 @@ def save_campaign(self, campaign: ExperimentCampaign) -> str: ).fetchone() if existing is not None: existing_definition = json.loads(existing["definition_json"]) - if existing_definition != campaign.to_dict(): + incoming_definition = campaign.to_dict() + # Creation time is not part of deterministic campaign identity. + # Preserve the original definition while appending new executions. + existing_definition.pop("created_at", None) + incoming_definition.pop("created_at", None) + if existing_definition != incoming_definition: raise DuplicateRunError( f"Campaign ID {campaign_id} already exists with different content" ) @@ -93,14 +98,14 @@ def get_campaign(self, campaign_id: str) -> ExperimentCampaign: raise ValidationError(f"Campaign not found: {campaign_id}") return ExperimentCampaign.from_dict(json.loads(row["definition_json"])) - def list_campaigns(self, limit: int = 100) -> list[dict[str, Any]]: + def list_campaigns(self, limit: int = 100, offset: int = 0) -> list[dict[str, Any]]: if limit < 1 or limit > 1000: raise ValidationError("Campaign list limit must be between 1 and 1000") with self._connection() as connection: rows = connection.execute( "SELECT campaign_id, name, family, created_at FROM campaigns " - "ORDER BY created_at DESC, rowid DESC LIMIT ?", - (limit,), + "ORDER BY created_at DESC, rowid DESC LIMIT ? OFFSET ?", + (limit, offset), ).fetchall() return [dict(row) for row in rows] @@ -131,15 +136,38 @@ def get_run_ids(self, campaign_id: str) -> list[str]: ).fetchall() return [row["run_id"] for row in rows] - def get_runs_with_status(self, campaign_id: str) -> list[dict[str, Any]]: + def get_runs_with_status( + self, campaign_id: str, *, limit: int | None = None, offset: int = 0 + ) -> list[dict[str, Any]]: with self._connection() as connection: + suffix = " LIMIT ? OFFSET ?" if limit is not None else "" + parameters = (campaign_id, limit, offset) if limit is not None else (campaign_id,) rows = connection.execute( "SELECT run_id, status, solver_name, problem_id, created_at " - "FROM campaign_runs WHERE campaign_id = ? ORDER BY created_at", - (campaign_id,), + "FROM campaign_runs WHERE campaign_id = ? ORDER BY created_at" + suffix, + parameters, ).fetchall() return [dict(row) for row in rows] + def run_statistics(self, campaign_id: str) -> dict[str, Any]: + """Aggregate membership without loading every run into presentation memory.""" + with self._connection() as connection: + rows = connection.execute( + "SELECT status, solver_name, COUNT(*) AS n FROM campaign_runs " + "WHERE campaign_id = ? GROUP BY status, solver_name", + (campaign_id,), + ).fetchall() + statuses: dict[str, int] = {} + solvers: dict[str, int] = {} + for row in rows: + statuses[row["status"]] = statuses.get(row["status"], 0) + row["n"] + solvers[row["solver_name"]] = solvers.get(row["solver_name"], 0) + row["n"] + return { + "completed_runs": sum(statuses.values()), + "status_counts": statuses, + "solver_breakdown": solvers, + } + def campaigns_for_run(self, run_id: str) -> list[dict[str, Any]]: with self._connection() as connection: rows = connection.execute( @@ -167,3 +195,17 @@ def campaigns_for_problem(self, problem_id: str) -> list[dict[str, Any]]: def count(self) -> int: with self._connection() as connection: return int(connection.execute("SELECT COUNT(*) FROM campaigns").fetchone()[0]) + + def fully_recorded_count(self) -> int: + """Membership coverage, not a claim of successful or recoverable execution.""" + with self._connection() as connection: + rows = connection.execute( + "SELECT c.definition_json, COUNT(cr.run_id) AS recorded FROM campaigns c " + "LEFT JOIN campaign_runs cr ON cr.campaign_id = c.campaign_id " + "GROUP BY c.campaign_id" + ).fetchall() + return sum( + row["recorded"] + >= ExperimentCampaign.from_dict(json.loads(row["definition_json"])).requested_runs + for row in rows + ) diff --git a/src/variaq/capabilities.py b/src/variaq/capabilities.py index 3a734d7..b0cfb7e 100644 --- a/src/variaq/capabilities.py +++ b/src/variaq/capabilities.py @@ -13,7 +13,6 @@ from typing import Any from variaq import __version__ -from variaq.errors import BackendUnavailableError from variaq.serialization import OUTPUT_SCHEMA_VERSION, StructuredWarning from variaq.solvers.base import get_solver, solver_names, solver_supported_families @@ -61,31 +60,26 @@ def _package_version(name: str) -> str | None: return None -def _solver_capability(name: str) -> CapabilityEntry: +def _solver_capability(name: str, cudaq_info: dict[str, Any] | None = None) -> CapabilityEntry: supported = name in solver_names() installed = False available = False reason: str | None = None try: - solver = get_solver(name) + get_solver(name) # Static adapter lookup; never execute a solver for discovery. installed = True - # Use a tiny synthetic problem to test availability without heavy work. - if name in {"qaoa", "cudaq-cpu", "cudaq-gpu"}: - from variaq.models import SolverConfig - from variaq.problems.maxcut import MaxCutProblem + if name == "qaoa": + from variaq.solvers.qiskit_qaoa import _load_quantum_dependencies - try: - solver.solve( - MaxCutProblem.from_edges(2, [(0, 1)]), - SolverConfig(seed=0, parameters={"optimizer_trials": 1, "shots": 1}), - ) - available = True - except BackendUnavailableError as exc: - reason = str(exc) - except Exception as exc: - # Missing dependency / limit / runtime problem means installed but - # not available for this environment. - reason = f"{type(exc).__name__}: {exc}" + installed = _package_version("qiskit") is not None + _load_quantum_dependencies() + available = True + elif name in {"cudaq-cpu", "cudaq-gpu"}: + info = cudaq_info if cudaq_info is not None else _cudaq_targets() + installed = info["installed"] + key = "qpp_cpu_available" if name == "cudaq-cpu" else "nvidia_available" + available = bool(info[key]) + reason = None if available else info.get("reason") or "CUDA-Q target is unavailable" else: available = True except Exception as exc: @@ -199,7 +193,7 @@ def gather_capabilities() -> dict[str, Any]: {"name": "subset-selection", "supported": True}, {"name": "graph-partition", "supported": True}, ], - "solvers": [_solver_capability(name).to_dict() for name in solver_names()], + "solvers": [_solver_capability(name, cudaq_info).to_dict() for name in solver_names()], "frameworks": [ FrameworkInfo( name="qiskit", version=qiskit_version, installed=qiskit_version is not None @@ -221,6 +215,9 @@ def gather_capabilities() -> dict[str, Any]: }, }, ], + "availability_scope": ( + "dependency_import_target_and_device_discovery_without_solver_execution" + ), "physical_qpu": physical_qpu, "warnings": [warning.to_dict() for warning in warnings], } diff --git a/src/variaq/cli.py b/src/variaq/cli.py index edc731f..4a61b74 100644 --- a/src/variaq/cli.py +++ b/src/variaq/cli.py @@ -31,7 +31,7 @@ solve_json_data, ) from variaq.problems.assignment import AssignmentProblem -from variaq.problems.base import ProblemInstance, load_problem, save_problem +from variaq.problems.base import ProblemInstance, load_problem, problem_from_dict, save_problem from variaq.problems.graph_partition import GraphPartitionProblem from variaq.problems.maxcut import MaxCutProblem from variaq.problems.subset_selection import SubsetSelectionProblem @@ -236,32 +236,19 @@ def _print_quantum_comparison(runs: list[ExperimentRun]) -> None: for seed, group in sorted(by_seed.items()): successful = [run for run in group if run.result.status is SolveStatus.SUCCESS] unavailable = [run for run in group if run.result.status is SolveStatus.UNAVAILABLE] - digests = { - str(run.result.backend.metrics.get("candidate_parameter_digest")) for run in successful - } - max_delta = 0.0 - if len(successful) > 1: - reference = successful[0].result.backend.metrics.get("candidate_expectations", []) - for run in successful[1:]: - candidate = run.result.backend.metrics.get("candidate_expectations", []) - if len(candidate) == len(reference): - max_delta = max( - max_delta, - max( - ( - abs(float(a) - float(b)) - for a, b in zip(reference, candidate, strict=True) - ), - default=0.0, - ), - ) + comparison = quantum_comparison_json_data(problem_from_dict(group[0].problem), group)[ + "comparison" + ] + max_delta = comparison["max_expectation_delta"] + identical = comparison["identical_candidate_parameters"] best_indices = { run.result.solver_name: run.result.backend.metrics.get("best_parameter_index") for run in successful } print( - f" seed={seed}: identical_candidates={'yes' if len(digests) <= 1 else 'NO'}, " - f"max_expectation_delta={max_delta:.3g}, best_parameter_indices={best_indices}" + f" seed={seed}: identical_candidates={identical}, " + f"max_expectation_delta={_format_number(max_delta)}, " + f"best_parameter_indices={best_indices}" ) for run in unavailable: reason = run.result.backend.metrics.get("reason", "unavailable") @@ -404,7 +391,7 @@ def _build_parser() -> argparse.ArgumentParser: "--compare", action="append", default=[], - choices=["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"], + choices=["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu", "paired_solvers"], ) report = subcommands.add_parser("report", help="Generate reports from analysis results") @@ -421,7 +408,7 @@ def _build_parser() -> argparse.ArgumentParser: "--compare", action="append", default=[], - choices=["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"], + choices=["classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu", "paired_solvers"], ) runs = subcommands.add_parser("runs", help="Inspect or reproduce durable experiment runs") @@ -964,6 +951,7 @@ def _command_report( run_ids=tuple(run_ids), group_by=tuple(args.group_by) if args.group_by else ("problem_id", "solver"), include_failed=True, + include_unavailable=True, ) comparisons = tuple(args.compare) if args.compare else () analysis = analyze_runs(runs, query, scaling_x_metric=args.scaling_x, comparisons=comparisons) diff --git a/src/variaq/experiments/runner.py b/src/variaq/experiments/runner.py index 5398d0c..bfa60c4 100644 --- a/src/variaq/experiments/runner.py +++ b/src/variaq/experiments/runner.py @@ -1,11 +1,16 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import replace from time import perf_counter from uuid import uuid4 -from variaq.errors import BackendUnavailableError, UnsafeBackendError, ValidationError +from variaq.errors import ( + BackendUnavailableError, + MissingOptionalDependency, + UnsafeBackendError, + ValidationError, +) from variaq.experiments.storage import ExperimentStore from variaq.models import ( BackendMetadata, @@ -32,6 +37,7 @@ class ExperimentRunner: def __init__(self, store: ExperimentStore) -> None: self.store = store + self._solver_invocations: dict[str, int] = {} def _execute( self, @@ -40,15 +46,16 @@ def _execute( config: SolverConfig, benchmark_id: str | None, rerun_of: str | None = None, + repeat_index: int | None = None, ) -> ExperimentRun: started = perf_counter() try: result = solver.solve(problem, config) if not result.backend.is_local: raise UnsafeBackendError( - "VariaQ v0.2 refuses non-local backends; no QPU execution is supported" + "VariaQ refuses non-local backends; no QPU execution is supported" ) - except BackendUnavailableError as exc: + except (BackendUnavailableError, MissingOptionalDependency) as exc: elapsed = perf_counter() - started result = SolveResult( solver_name=getattr(solver, "name", type(solver).__name__), @@ -64,8 +71,8 @@ def _execute( solver_time_seconds=None, backend=BackendMetadata( backend_type="unavailable", - name=exc.backend_name, - provider=exc.provider, + name=getattr(exc, "backend_name", solver.name), + provider=getattr(exc, "provider", "variaq"), is_local=True, metrics={"capability_available": False, "reason": str(exc)}, ), @@ -101,6 +108,34 @@ def _execute( status=SolveStatus.FAILED, errors=(f"{type(exc).__name__}: {exc}",), ) + # Measure the complete solver call, including validation/lowering/imports. + # Persistence and environment capture are outside this wall-time scope. + elapsed = perf_counter() - started + metrics = dict(result.backend.metrics) + metrics.update( + wall_time_scope="solver_call_including_setup_excluding_persistence", + solver_time_scope=( + "warmup_expectations_and_sampling" + if solver.name in QAOA_SOLVER_NAMES + else "solver_reported_compute" + ), + ) + result = replace( + result, + wall_time_seconds=elapsed, + backend=replace(result.backend, metrics=metrics), + ) + invocation = self._solver_invocations.get(solver.name, 0) + self._solver_invocations[solver.name] = invocation + 1 + environment = capture_environment() + environment["benchmark_protocol"] = { + "repeat_index": repeat_index, + "solver_invocation_index": invocation, + "warmup_requested": config.parameters.get("warmup", False), + "process_isolated": False, + "cold_start_verified": None, + "cold_start_reason": "Runner reuse does not establish fresh backend/device state", + } return ExperimentRun( run_id=new_run_id(), benchmark_id=benchmark_id, @@ -108,7 +143,7 @@ def _execute( problem=problem.to_dict(), solver_config=config, result=result, - environment=capture_environment(), + environment=environment, rerun_of=rerun_of, ) @@ -132,12 +167,20 @@ def _add_comparison( ) if problem.sense is OptimizationSense.MAXIMIZE: delta = best_known - result.objective - gap = max(0.0, delta / abs(best_known) * 100.0) if best_known != 0 else 0.0 + gap = ( + max(0.0, delta / abs(best_known) * 100.0) + if best_known != 0 + else (0.0 if delta == 0 else None) + ) ratio = result.objective / best_known if best_known > 0 else None normalized = ratio else: delta = result.objective - best_known - gap = max(0.0, delta / abs(best_known) * 100.0) if best_known != 0 else 0.0 + gap = ( + max(0.0, delta / abs(best_known) * 100.0) + if best_known != 0 + else (0.0 if delta == 0 else None) + ) ratio = best_known / result.objective if result.objective > 0 else None normalized = ratio return replace( @@ -146,7 +189,9 @@ def _add_comparison( result, best_known_objective=best_known, best_known_source=source, - optimality_gap_percent=gap, + optimality_gap_percent=gap + if source in {"exact_optimum", "stored_exact_optimum"} + else None, approximation_ratio=ratio, normalized_score=normalized, ), @@ -159,8 +204,11 @@ def run_one( config: SolverConfig, *, rerun_of: str | None = None, + web_task_id: str | None = None, ) -> ExperimentRun: run = self._execute(problem, solver, config, None, rerun_of) + if web_task_id is not None: + run = replace(run, environment={**run.environment, "web_task_id": web_task_id}) if run.result.status is SolveStatus.SUCCESS and solver.name == "exact": best_known = run.result.objective source = "exact_optimum" @@ -177,6 +225,11 @@ def benchmark( solvers: Iterable[Solver], configs: dict[str, SolverConfig], repeats: int = 1, + *, + on_persisted: Callable[[ExperimentRun], None] | None = None, + campaign_id: str | None = None, + web_task_id: str | None = None, + should_abort: Callable[[], bool] | None = None, ) -> list[ExperimentRun]: if repeats < 1: raise ValueError("repeats must be positive") @@ -190,7 +243,31 @@ def benchmark( for solver in solver_list: for repeat in range(repeats): repeat_config = repeat_configs[repeat][solver.name] - pending.append(self._execute(problem, solver, repeat_config, benchmark_id)) + if should_abort is not None and should_abort(): + return pending + run = self._execute( + problem, solver, repeat_config, benchmark_id, repeat_index=repeat + ) + if on_persisted is not None: + # Campaigns persist each completed attempt before starting another. + # Later exact references are derived by analysis, never by rewriting history. + exact = self.store.exact_objective(problem.problem_id) + source = "stored_exact_optimum" if exact is not None else None + if solver.name == "exact" and run.result.feasible: + exact, source = run.result.objective, "exact_optimum" + run = self._add_comparison(run, problem, exact, source) + environment = { + **run.environment, + "campaign_id": campaign_id, + "web_task_id": web_task_id, + } + run = replace(run, environment=environment) + self.store.save(run) + on_persisted(run) + pending.append(run) + + if on_persisted is not None: + return pending exact_values = [ run.result.objective @@ -202,7 +279,11 @@ def benchmark( ] stored_exact = self.store.exact_objective(problem.problem_id) if exact_values: - best_known = max(exact_values) + best_known = ( + max(exact_values) + if problem.sense is OptimizationSense.MAXIMIZE + else min(exact_values) + ) source = "exact_optimum" elif stored_exact is not None: best_known = stored_exact @@ -215,7 +296,11 @@ def benchmark( and run.result.feasible and run.result.objective is not None ] - best_known = max(observed) if observed else None + best_known = ( + (max(observed) if problem.sense is OptimizationSense.MAXIMIZE else min(observed)) + if observed + else None + ) source = "best_observed_not_proven_optimal" if observed else None completed = [self._add_comparison(run, problem, best_known, source) for run in pending] @@ -284,7 +369,13 @@ def _matched_repeat_configs( else: # Generate once per repeat, then place the identical immutable sequence into # every solver configuration and persisted experiment record. - candidates = generate_parameter_candidates(p, optimizer_trials, reference_seed) + try: + candidates = generate_parameter_candidates(p, optimizer_trials, reference_seed) + except MissingOptionalDependency: + # Optional candidate-generation dependencies must not prevent + # classical baselines from running. Quantum solvers will record + # their own unavailable result through the normal executor. + return prepared serialized = [list(candidate) for candidate in candidates] for name in quantum_names: @@ -301,7 +392,7 @@ def _matched_repeat_configs( prepared[name] = SolverConfig(seed=reference_seed, parameters=parameters) return prepared - def reproduce(self, run_id: str) -> ExperimentRun: + def reproduce(self, run_id: str, *, web_task_id: str | None = None) -> ExperimentRun: original = self.store.get(run_id) problem = problem_from_dict(original.problem) solver = get_solver(original.result.solver_name) @@ -310,4 +401,5 @@ def reproduce(self, run_id: str) -> ExperimentRun: solver, original.solver_config, rerun_of=original.run_id, + web_task_id=web_task_id, ) diff --git a/src/variaq/experiments/storage.py b/src/variaq/experiments/storage.py index 8073f67..0580730 100644 --- a/src/variaq/experiments/storage.py +++ b/src/variaq/experiments/storage.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math import sqlite3 from collections.abc import Iterator from contextlib import contextmanager @@ -198,6 +199,7 @@ def query_runs( status: str | None = None, problem_id: str | None = None, run_ids: set[str] | frozenset[str] | None = None, + campaign_id: str | None = None, limit: int = 50, offset: int = 0, ) -> tuple[list[dict[str, Any]], int]: @@ -234,6 +236,12 @@ def query_runs( placeholders = ", ".join("?" for _ in run_ids) clauses.append(f"run_id IN ({placeholders})") params.extend(sorted(run_ids)) + if campaign_id is not None: + clauses.append( + "EXISTS (SELECT 1 FROM campaign_runs cr " + "WHERE cr.run_id = runs.run_id AND cr.campaign_id = ?)" + ) + params.append(campaign_id) where = f" WHERE {' AND '.join(clauses)}" if clauses else "" with self._connection() as connection: total = int( @@ -313,16 +321,28 @@ def get_problem_definition(self, problem_id: str) -> dict[str, Any]: return json.loads(row["definition_json"]) def exact_objective(self, problem_id: str) -> float | None: + """Return a consistent canonical feasible exact reference, for either sense.""" with self._connection() as connection: - row = connection.execute( + rows = connection.execute( """ - SELECT MAX(objective) AS objective FROM runs + SELECT record_json FROM runs WHERE problem_id = ? AND solver_name = 'exact' AND status = 'success' AND feasible = 1 """, (problem_id,), - ).fetchone() - return None if row is None else row["objective"] + ).fetchall() + objectives: set[float] = set() + for row in rows: + run = ExperimentRun.from_dict(json.loads(row["record_json"])) + problem = problem_from_dict(run.problem) + if run.result.solution is None: + continue + evaluation = problem.evaluate(run.result.solution) + if evaluation.feasible and evaluation.objective is not None: + objective = float(evaluation.objective) + if math.isfinite(objective): + objectives.add(objective) + return next(iter(objectives)) if len(objectives) == 1 else None def count(self) -> int: with self._connection() as connection: diff --git a/src/variaq/outputs.py b/src/variaq/outputs.py index 6412106..b4c1dca 100644 --- a/src/variaq/outputs.py +++ b/src/variaq/outputs.py @@ -4,6 +4,7 @@ from typing import Any +from variaq.analysis.paired import paired_analysis from variaq.models import ExperimentRun, SolveResult, SolveStatus from variaq.serialization import OUTPUT_SCHEMA_VERSION, StructuredWarning @@ -117,33 +118,28 @@ def quantum_comparison_json_data(problem: Any, runs: list[ExperimentRun]) -> dic base = benchmark_json_data(problem, runs) successful = [run for run in runs if run.result.status is SolveStatus.SUCCESS] unavailable = [run for run in runs if run.result.status is SolveStatus.UNAVAILABLE] - max_delta = 0.0 - identical_candidates = True - reference_digest: str | None = None - if successful: - reference_digest = str( - successful[0].result.backend.metrics.get("candidate_parameter_digest") - ) - digests = { - str(run.result.backend.metrics.get("candidate_parameter_digest")) for run in successful + _, paired = paired_analysis(runs) + paired_repeats = [ + { + "benchmark_id": p["benchmark_id"], + "repeat_key": p["repeat_key"], + "left_run_id": p["left_run_id"], + "right_run_id": p["right_run_id"], + **p["numerical_correctness"], } - identical_candidates = len(digests) <= 1 - reference_expectations = list( - successful[0].result.backend.metrics.get("candidate_expectations", []) - ) - for run in successful[1:]: - candidate = list(run.result.backend.metrics.get("candidate_expectations", [])) - if len(candidate) == len(reference_expectations): - max_delta = max( - max_delta, - max( - ( - abs(float(a) - float(b)) - for a, b in zip(reference_expectations, candidate, strict=True) - ), - default=0.0, - ), - ) + for p in paired.pairs + ] + deltas = [ + p["max_expectation_delta"] for p in paired_repeats if p["max_expectation_delta"] is not None + ] + max_delta = max(deltas) if deltas else None + controls = [p["matched_controls"] for p in paired_repeats if p["matched_controls"] is not None] + identical_candidates = all(controls) if controls else None + reference_digest = ( + successful[0].result.backend.metrics.get("candidate_parameter_digest") + if successful + else None + ) best_indices = { run.result.solver_name: run.result.backend.metrics.get("best_parameter_index") for run in successful @@ -166,6 +162,9 @@ def quantum_comparison_json_data(problem: Any, runs: list[ExperimentRun]) -> dic "candidate_parameter_digest": reference_digest, "identical_candidate_parameters": identical_candidates, "max_expectation_delta": max_delta, + "expectation_comparison_count": len(deltas), + "expectation_comparison_reason": None if deltas else "No complete matched backend pair", + "paired_repeats": paired_repeats, "best_parameter_indices": best_indices, "precision": precision_map, "backend_target": target_map, diff --git a/src/variaq/services.py b/src/variaq/services.py index 3cb8b4d..9a4ef86 100644 --- a/src/variaq/services.py +++ b/src/variaq/services.py @@ -17,6 +17,7 @@ from typing import Any from variaq import __version__ +from variaq.analysis.comparison_view import comparison_view from variaq.analysis.models import AnalysisQuery from variaq.analysis.operations import analyze_runs from variaq.analysis.reports import export_report, generate_report @@ -78,8 +79,9 @@ def overview(self) -> dict[str, Any]: "campaigns": self.campaign_store.count(), "reports": len(self.list_reports()["reports"]), }, + "fully_recorded_campaigns": self.campaign_store.fully_recorded_count(), "recent_runs": self.list_runs(limit=5)["runs"], - "recent_campaigns": self.campaign_store.list_campaigns(limit=5), + "recent_campaigns": self.list_campaigns(limit=5)["campaigns"], "solver_availability": { name: {"available": s["available"], "installed": s["installed"]} for name, s in solvers.items() @@ -168,25 +170,21 @@ def list_runs( _check_limit_offset(limit, offset) if status is not None and status not in {"success", "failed", "unavailable"}: raise ValidationError(f"Invalid status filter: {status!r}") - campaign_members: set[str] | None = None if campaign_id is not None: # Validates the campaign exists. self.campaign_store.get_campaign(campaign_id) - campaign_members = set(self.campaign_store.get_run_ids(campaign_id)) rows, total = self.store.query_runs( problem_type=family, solver_name=solver, backend_name=backend, status=status, problem_id=problem_id, - run_ids=campaign_members, + campaign_id=campaign_id, limit=limit, offset=offset, ) for row in rows: - row["in_campaign"] = bool( - campaign_members is not None and row["run_id"] in campaign_members - ) + row["in_campaign"] = campaign_id is not None return {"total": total, "limit": limit, "offset": offset, "runs": rows} def get_run(self, run_id: str) -> dict[str, Any]: @@ -195,52 +193,168 @@ def get_run(self, run_id: str) -> dict[str, Any]: data = run.to_dict() data["campaigns"] = campaigns data["rerun_of"] = run.rerun_of + data["reproduction_eligible"] = run.result.backend.is_local return data + def reproduce(self, run_id: str, *, web_task_id: str | None = None) -> dict[str, Any]: + original = self.store.get(run_id) + if not original.result.backend.is_local: + raise ValidationError("Only local experiments can be reproduced") + return ExperimentRunner(self.store).reproduce(run_id, web_task_id=web_task_id).to_dict() + + def compare(self, selection: dict[str, Any], *, export: bool = False) -> dict[str, Any]: + """Analyze one bounded window. Never silently treat a window as the whole campaign.""" + campaign_id = selection.get("campaign_id") + ids = selection.get("run_ids") + if bool(campaign_id) == bool(ids): + raise ValidationError("Select either one campaign or explicit run IDs") + if campaign_id: + self.campaign_store.get_campaign(campaign_id) + if ids and len(set(ids)) != len(ids): + raise ValidationError("Duplicate run IDs are not allowed") + if ids: + for rid in ids: + self.store.get(rid) # Missing explicit IDs must not silently disappear. + rows, total = self.store.query_runs( + campaign_id=campaign_id, + run_ids=set(ids) if ids else None, + problem_id=selection.get("problem_id"), + solver_name=selection.get("solver"), + backend_name=selection.get("backend"), + limit=selection.get("limit", 500), + offset=selection.get("offset", 0), + ) + runs = [self.store.get(row["run_id"]) for row in rows] + # Size and warmup require canonical problem/configuration data. Apply only + # within this explicitly labelled bounded window, before computing references. + from variaq.problems.base import problem_from_dict + from variaq.solvers.qaoa_shared import QAOA_SOLVER_NAMES + + if selection.get("size") is not None: + runs = [ + run + for run in runs + if problem_from_dict(run.problem).variable_count == selection["size"] + ] + if selection.get("warmup"): + + def mode(run: Any) -> str: + if run.result.solver_name not in QAOA_SOLVER_NAMES: + return "not_applicable" + value = run.solver_config.parameters.get("warmup") + return ( + "explicit_warmup" + if value is True + else "unwarmed" + if value is False + else "unknown" + ) + + runs = [run for run in runs if mode(run) == selection["warmup"]] + query = AnalysisQuery( + campaign_id=campaign_id, + run_ids=tuple(run.run_id for run in runs), + group_by=("problem_id", "solver", "backend", "parameters.warmup"), + include_failed=True, + include_unavailable=True, + ) + analysis = analyze_runs( + runs, query, scaling_x_metric="problem_size", comparisons=("paired_solvers",) + ) + result = { + "analysis": analysis.to_dict(), + "instance_summaries": comparison_view(analysis.instances), + "total": total, + "window_count": len(rows), + "selected_count": len(runs), + "limit": selection.get("limit", 500), + "offset": selection.get("offset", 0), + "selection": selection, + "repeat_scaling": [ + { + "problem_id": instance["problem_id"], + "size": problem_from_dict(instance["problem"]).variable_count, + "solver": repeat["solver"], + "backend": repeat["backend"], + "parameters": repeat["parameters"], + "warmup_mode": repeat["warmup_mode"], + "wall": repeat["timing"]["wall_time_seconds"], + } + for instance in analysis.instances + for repeat in instance["repeats"] + ], + } + if export: + if not runs: + raise ValidationError("No runs selected for export") + report = generate_report(analysis, campaign_id=campaign_id) + # Presentation exports use fixed artifact names only. In particular, + # never follow a planted symlink when regenerating a deterministic ID. + for suffix in ( + ".json", + ".md", + "_groups.csv", + "_scaling.csv", + "_runs.csv", + "_pairs.csv", + "_repeats.csv", + ): + target = self.reports_dir / f"{report.report_id}{suffix}" + if target.is_symlink() or not self._is_within_root( + target, self.reports_dir.resolve() + ): + raise ValidationError( + "Requested file is outside the report root or is a symlink" + ) + existing = self.reports_dir / f"{report.report_id}.json" + if existing.exists(): + saved = json.loads(existing.read_text(encoding="utf-8")) + if saved.get("analysis") != analysis.to_dict(): + raise ValidationError("Existing report content differs from selected evidence") + report.generated_at = saved["generated_at"] + export_report( + report, self.reports_dir, formats=("json", "csv", "markdown"), overwrite=True + ) + result["report_id"] = report.report_id + return result + # -- campaigns ----------------------------------------------------------------- def list_campaigns(self, *, limit: int = 100, offset: int = 0) -> dict[str, Any]: _check_limit_offset(limit, offset) - # CampaignStore.list_campaigns is newest-first and limit-bounded; we - # paginate by fetching a larger window when an offset is requested. - rows = self.campaign_store.list_campaigns(limit=MAX_LIMIT) + rows = self.campaign_store.list_campaigns(limit=limit, offset=offset) decorated = [] for row in rows: - member_rows = self.campaign_store.get_runs_with_status(row["campaign_id"]) - statuses: dict[str, int] = {} - for member in member_rows: - statuses[member["status"]] = statuses.get(member["status"], 0) + 1 + stats = self.campaign_store.run_statistics(row["campaign_id"]) definition = self.campaign_store.get_campaign(row["campaign_id"]) decorated.append( { **row, "requested_runs": definition.requested_runs, - "completed_runs": len(member_rows), - "status_counts": statuses, + "completed_runs": stats["completed_runs"], + "status_counts": stats["status_counts"], } ) return { - "total": len(decorated), + "total": self.campaign_store.count(), "limit": limit, "offset": offset, - "campaigns": decorated[offset : offset + limit], + "campaigns": decorated, } - def get_campaign(self, campaign_id: str) -> dict[str, Any]: + def get_campaign( + self, campaign_id: str, *, limit: int | None = None, offset: int = 0 + ) -> dict[str, Any]: + if limit is not None: + _check_limit_offset(limit, offset) campaign = self.campaign_store.get_campaign(campaign_id) - members = self.campaign_store.get_runs_with_status(campaign_id) - statuses: dict[str, int] = {} - solvers: dict[str, int] = {} - for member in members: - statuses[member["status"]] = statuses.get(member["status"], 0) + 1 - solvers[member["solver_name"]] = solvers.get(member["solver_name"], 0) + 1 + members = self.campaign_store.get_runs_with_status(campaign_id, limit=limit, offset=offset) + stats = self.campaign_store.run_statistics(campaign_id) return { "definition": campaign.to_dict(), "campaign_id": campaign_id, "requested_runs": campaign.requested_runs, - "completed_runs": len(members), - "status_counts": statuses, - "solver_breakdown": solvers, + **stats, "runs": members, } @@ -258,6 +372,7 @@ def run_campaign( max_runs: int = DEFAULT_MAX_RUNS, on_run_recorded: Any | None = None, should_abort: Any | None = None, + web_task_id: str | None = None, ) -> dict[str, Any]: """Execute a campaign. Must be called from a worker thread by the web app.""" campaign = ExperimentCampaign.from_dict(definition) @@ -273,6 +388,7 @@ def run_campaign( override_max_runs=override_max_runs, on_run_recorded=on_run_recorded, should_abort=should_abort, + web_task_id=web_task_id, ) # -- analysis ------------------------------------------------------------------ @@ -381,6 +497,9 @@ def _report_file_format(report_id: str, name: str) -> str | None: ".md": "markdown", "_groups.csv": "csv", "_scaling.csv": "csv", + "_runs.csv": "csv", + "_pairs.csv": "csv", + "_repeats.csv": "csv", "_scaling.png": "plot", } for suffix, report_format in suffixes.items(): @@ -461,6 +580,7 @@ def create_report( run_ids=tuple(run_ids), group_by=tuple(group_by or ("problem_id", "solver")), include_failed=True, + include_unavailable=True, ) analysis = analyze_runs( runs, diff --git a/src/variaq/solvers/cudaq_qaoa.py b/src/variaq/solvers/cudaq_qaoa.py index eb9ee64..3ce3ea9 100644 --- a/src/variaq/solvers/cudaq_qaoa.py +++ b/src/variaq/solvers/cudaq_qaoa.py @@ -45,9 +45,12 @@ def _load_cudaq() -> Any: @contextmanager -def _selected_target(cudaq: Any, target_name: str) -> Any: +def _selected_target(cudaq: Any, target_name: str, precision: str | None = None) -> Any: previous = cudaq.get_target() - cudaq.set_target(target_name) + if target_name == "nvidia" and precision is not None: + cudaq.set_target(target_name, option=precision) + else: + cudaq.set_target(target_name) try: yield finally: @@ -78,10 +81,14 @@ def _optional_gpu_metadata() -> dict[str, Any]: for line in result.stdout.splitlines(): values = [value.strip() for value in line.split(",")] if len(values) == 3: + try: + memory_mib = int(values[1]) + except ValueError: + memory_mib = None devices.append( { "model": values[0], - "total_memory_mib": int(values[1]), + "total_memory_mib": memory_mib, "driver_version": values[2], } ) @@ -197,7 +204,9 @@ def _precision_and_target(self, parameters: dict[str, Any]) -> tuple[str, str]: if precision != "fp64": raise ValidationError("qpp-cpu supports VariaQ's fp64 mode only") return precision, "qpp-cpu" - return precision, "nvidia" if precision == "fp32" else "nvidia-fp64" + # The deprecated nvidia-fp64 alias prints native diagnostics to stdout, + # corrupting schema-v1 JSON. Select precision through supported options. + return precision, "nvidia" def solve(self, problem: ProblemInstance, config: SolverConfig) -> SolveResult: self.check_family(problem) @@ -263,7 +272,7 @@ def solve(self, problem: ProblemInstance, config: SolverConfig) -> SolveResult: provider="cudaq", ) - with _selected_target(cudaq, target_name): + with _selected_target(cudaq, target_name, precision): backend = CudaQStatevectorBackend(cudaq, bqm, p, config.seed) initialization_seconds = perf_counter() - initialization_started outcome = run_shared_parameter_search( @@ -296,6 +305,7 @@ def solve(self, problem: ProblemInstance, config: SolverConfig) -> SolveResult: metrics: dict[str, Any] = { "framework": "cudaq", "target_name": target_name, + "target_options": {"precision": precision}, "target_type": f"{self.target_kind.upper()} simulator", "floating_point_precision": precision, "simulator_backend": target_name, @@ -321,6 +331,10 @@ def solve(self, problem: ProblemInstance, config: SolverConfig) -> SolveResult: "best_solution_energy": outcome.best_solution_energy, "best_infeasible_energy": outcome.best_infeasible_energy, "backend_initialization_seconds": initialization_seconds, + "backend_initialization_scope": "imports_target_selection_kernel_construction", + "first_backend_evaluation_seconds": outcome.first_backend_evaluation_seconds, + "first_backend_evaluation_scope": "first_observe_including_lazy_backend_setup", + "first_backend_evaluation_overlaps": "warmup_or_expectation_evaluation_seconds", "warmup_seconds": outcome.warmup_seconds, "parameter_search_seconds": outcome.parameter_search_seconds, "expectation_evaluation_seconds": outcome.expectation_evaluation_seconds, diff --git a/src/variaq/solvers/qaoa_shared.py b/src/variaq/solvers/qaoa_shared.py index fa0cc79..fbb742e 100644 --- a/src/variaq/solvers/qaoa_shared.py +++ b/src/variaq/solvers/qaoa_shared.py @@ -197,6 +197,7 @@ class QAOASearchOutcome: parameter_search_seconds: float expectation_evaluation_seconds: float final_sampling_seconds: float + first_backend_evaluation_seconds: float def run_shared_parameter_search( @@ -215,10 +216,12 @@ def run_shared_parameter_search( raise ValidationError("shots must be positive") warmup_seconds = 0.0 + first_backend_evaluation_seconds = None if warmup: warmup_started = perf_counter() evaluate_expectation(candidates[0]) warmup_seconds = perf_counter() - warmup_started + first_backend_evaluation_seconds = warmup_seconds search_started = perf_counter() expectation_seconds = 0.0 @@ -226,7 +229,10 @@ def run_shared_parameter_search( for index, candidate in enumerate(candidates): evaluation_started = perf_counter() expectation = float(evaluate_expectation(candidate)) - expectation_seconds += perf_counter() - evaluation_started + evaluation_seconds = perf_counter() - evaluation_started + expectation_seconds += evaluation_seconds + if first_backend_evaluation_seconds is None: + first_backend_evaluation_seconds = evaluation_seconds if not math.isfinite(expectation): raise ValidationError( f"Backend returned a non-finite expectation for candidate {index}" @@ -306,6 +312,7 @@ def run_shared_parameter_search( parameter_search_seconds=search_seconds, expectation_evaluation_seconds=expectation_seconds, final_sampling_seconds=sampling_seconds, + first_backend_evaluation_seconds=first_backend_evaluation_seconds, ) final_evaluation = problem.evaluate(qaoa.bqm.decode_bits(best_feasible_solution)) @@ -328,6 +335,7 @@ def run_shared_parameter_search( parameter_search_seconds=search_seconds, expectation_evaluation_seconds=expectation_seconds, final_sampling_seconds=sampling_seconds, + first_backend_evaluation_seconds=first_backend_evaluation_seconds, ) diff --git a/src/variaq/solvers/qiskit_qaoa.py b/src/variaq/solvers/qiskit_qaoa.py index e0c0ae7..fc76001 100644 --- a/src/variaq/solvers/qiskit_qaoa.py +++ b/src/variaq/solvers/qiskit_qaoa.py @@ -194,6 +194,10 @@ def solve(self, problem: ProblemInstance, config: SolverConfig) -> SolveResult: "best_solution_energy": outcome.best_solution_energy, "best_infeasible_energy": outcome.best_infeasible_energy, "backend_initialization_seconds": initialization_seconds, + "backend_initialization_scope": "imports_and_backend_construction", + "first_backend_evaluation_seconds": outcome.first_backend_evaluation_seconds, + "first_backend_evaluation_scope": "first_statevector_expectation_call", + "first_backend_evaluation_overlaps": "warmup_or_expectation_evaluation_seconds", "warmup_seconds": outcome.warmup_seconds, "parameter_search_seconds": outcome.parameter_search_seconds, "expectation_evaluation_seconds": outcome.expectation_evaluation_seconds, diff --git a/src/variaq/web/app.py b/src/variaq/web/app.py index 694b503..973e4c8 100644 --- a/src/variaq/web/app.py +++ b/src/variaq/web/app.py @@ -12,8 +12,10 @@ from __future__ import annotations import logging +from contextlib import asynccontextmanager from pathlib import Path from typing import Any +from urllib.parse import urlsplit from fastapi import FastAPI, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse @@ -27,7 +29,9 @@ AnalysisRequest, CampaignPlanRequest, CampaignRunRequest, + ComparisonRequest, ReportRequest, + ReproductionRequest, ) from variaq.web.tasks import CampaignTaskManager @@ -44,16 +48,73 @@ def create_app( service: VariaQService, *, task_manager: CampaignTaskManager | None = None, + trusted_hosts: tuple[str, ...] = ("127.0.0.1", "localhost", "::1", "testserver"), ) -> FastAPI: + tasks = task_manager or CampaignTaskManager() + tasks.bind(service) + + @asynccontextmanager + async def lifespan(_app: FastAPI): + yield + tasks.close() + app = FastAPI( title="VariaQ", version="1", docs_url=None, redoc_url=None, openapi_url=None, + lifespan=lifespan, ) app.state.service = service - tasks = task_manager or CampaignTaskManager() + + @app.middleware("http") + async def same_origin_writes(request: Request, call_next: Any) -> Any: + if "*" not in trusted_hosts: + host = request.headers.get("host", "") + try: + parsed_host = urlsplit("//" + host) + valid_host = ( + parsed_host.hostname in trusted_hosts + and parsed_host.username is None + and not any(c in host for c in "/\\?# \t\r\n") + ) + _ = parsed_host.port # Reject malformed port syntax as well. + except ValueError: + valid_host = False + if not valid_host: + return JSONResponse( + status_code=400, + content={ + "error": {"type": "InvalidHost", "message": "Untrusted local Host header"} + }, + ) + if request.method not in {"GET", "HEAD", "OPTIONS"}: + origin = request.headers.get("origin") + try: + parsed = urlsplit(origin) if origin else None + except ValueError: + return JSONResponse( + status_code=403, + content={ + "error": {"type": "CrossOriginWrite", "message": "Malformed Origin header"} + }, + ) + if request.headers.get("sec-fetch-site") == "cross-site" or ( + parsed is not None + and (parsed.scheme != request.url.scheme or parsed.netloc != request.url.netloc) + ): + return JSONResponse( + status_code=403, + content={ + "error": { + "type": "CrossOriginWrite", + "message": "Cross-origin execution is refused", + } + }, + ) + return await call_next(request) + app.state.tasks = tasks templates = Jinja2Templates(directory=str(TEMPLATE_DIR)) @@ -186,6 +247,23 @@ def api_runs( def api_run(run_id: str) -> dict[str, Any]: return svc().get_run(run_id) + @app.post(f"{API_PREFIX}/runs/{{run_id}}/reproduce") + def api_reproduce( + run_id: str, payload: ReproductionRequest, request: Request + ) -> dict[str, Any]: + task = request.app.state.tasks.start_reproduction(svc(), run_id) + if task is None: + raise HTTPException(status_code=409, detail="A local experiment is already running") + return {"task": task.to_dict()} + + @app.post(f"{API_PREFIX}/comparison") + def api_comparison(payload: ComparisonRequest) -> dict[str, Any]: + return svc().compare(payload.model_dump(exclude_none=True)) + + @app.post(f"{API_PREFIX}/comparison/reports") + def api_comparison_report(payload: ComparisonRequest) -> dict[str, Any]: + return svc().compare(payload.model_dump(exclude_none=True), export=True) + # -- API: campaigns ------------------------------------------------------------ @app.get(f"{API_PREFIX}/campaigns") @@ -208,8 +286,8 @@ def api_campaign_task(task_id: str, request: Request) -> dict[str, Any]: return {"task": task.to_dict()} @app.get(f"{API_PREFIX}/campaigns/{{campaign_id}}") - def api_campaign(campaign_id: str) -> dict[str, Any]: - return svc().get_campaign(campaign_id) + def api_campaign(campaign_id: str, limit: int | None = None, offset: int = 0) -> dict[str, Any]: + return svc().get_campaign(campaign_id, limit=limit, offset=offset) @app.post(f"{API_PREFIX}/campaigns/plan") def api_campaign_plan(payload: CampaignPlanRequest) -> dict[str, Any]: @@ -303,6 +381,10 @@ def page(request: Request, name: str, **context: Any) -> Any: def dashboard(request: Request) -> Any: return page(request, "dashboard.html") + @app.get("/compare", include_in_schema=False) + def comparison(request: Request) -> Any: + return page(request, "comparison.html") + @app.get("/problems", include_in_schema=False) def problems(request: Request) -> Any: return page(request, "problems.html") diff --git a/src/variaq/web/schemas.py b/src/variaq/web/schemas.py index ebc3857..22dea08 100644 --- a/src/variaq/web/schemas.py +++ b/src/variaq/web/schemas.py @@ -13,7 +13,41 @@ from variaq.campaigns.model import CAMPAIGN_FORMAT_VERSION, DEFAULT_MAX_RUNS -ALLOWED_COMPARISONS = {"classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu"} +ALLOWED_COMPARISONS = {"classical_vs_quantum", "qiskit_vs_cudaq", "gpu_vs_cpu", "paired_solvers"} + + +class ReproductionRequest(BaseModel): + """JSON-only intent; no arbitrary execution configuration is accepted.""" + + model_config = ConfigDict(extra="forbid") + + +class SolverOverrideIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + seed: int | None = None + parameters: dict[str, Any] = Field(default_factory=dict, max_length=32) + + @field_validator("parameters") + @classmethod + def budgets_bounded(cls, value: dict[str, Any]) -> dict[str, Any]: + ceilings = { + "p": 8, + "optimizer_trials": 10000, + "shots": 1000000, + "restarts": 10000, + "max_passes": 10000, + } + for key, ceiling in ceilings.items(): + if key in value: + number = value[key] + if ( + isinstance(number, bool) + or not isinstance(number, int) + or not 1 <= number <= ceiling + ): + raise ValueError(f"{key} must be an integer between 1 and {ceiling}") + return value class CampaignDefinitionIn(BaseModel): @@ -29,6 +63,7 @@ class CampaignDefinitionIn(BaseModel): repeats: int = Field(default=1, ge=1, le=100) base_seed: int = Field(default=0) generator_parameters: dict[str, Any] = Field(default_factory=dict) + solver_config: dict[str, SolverOverrideIn] = Field(default_factory=dict, max_length=20) tags: list[str] = Field(default_factory=list, max_length=20) notes: str = Field(default="", max_length=4000) campaign_format_version: str = CAMPAIGN_FORMAT_VERSION @@ -36,8 +71,18 @@ class CampaignDefinitionIn(BaseModel): @field_validator("problem_sizes") @classmethod def sizes_positive(cls, value: list[int]) -> list[int]: - if any(v < 1 for v in value): - raise ValueError("problem_sizes must be positive integers") + if any(not 1 <= v <= 256 for v in value): + raise ValueError("Web problem_sizes must be integers between 1 and 256") + return value + + @field_validator("generator_parameters") + @classmethod + def dimensions_bounded(cls, value: dict[str, Any]) -> dict[str, Any]: + for key in ("resource_count", "partition_count"): + if key in value: + count = value[key] + if isinstance(count, bool) or not isinstance(count, int) or not 1 <= count <= 256: + raise ValueError(f"Web {key} must be an integer between 1 and 256") return value def to_definition(self) -> dict[str, Any]: @@ -51,6 +96,10 @@ def to_definition(self) -> dict[str, Any]: "repeats": self.repeats, "base_seed": self.base_seed, "generator_parameters": dict(self.generator_parameters), + "solver_config": { + name: override.model_dump(exclude_none=True) + for name, override in self.solver_config.items() + }, "tags": list(self.tags), "notes": self.notes, } @@ -108,3 +157,18 @@ def comparisons_known(cls, value: list[str]) -> list[str]: if unknown: raise ValueError(f"Unknown comparisons: {sorted(unknown)}") return value + + +class ComparisonRequest(BaseModel): + """A bounded presentation query; scientific analysis stays in Python.""" + + model_config = ConfigDict(extra="forbid") + campaign_id: str | None = Field(default=None, max_length=128) + run_ids: list[str] | None = Field(default=None, min_length=1, max_length=500) + problem_id: str | None = Field(default=None, max_length=128) + solver: str | None = Field(default=None, max_length=64) + backend: str | None = Field(default=None, max_length=128) + size: int | None = Field(default=None, ge=1) + warmup: Literal["explicit_warmup", "unwarmed", "unknown", "not_applicable"] | None = None + limit: int = Field(default=500, ge=1, le=500) + offset: int = Field(default=0, ge=0) diff --git a/src/variaq/web/server.py b/src/variaq/web/server.py index 4cbce4d..c2b9658 100644 --- a/src/variaq/web/server.py +++ b/src/variaq/web/server.py @@ -13,7 +13,11 @@ def build_app(config: WebConfig): problems_dir=config.problems_dir, reports_dir=config.reports_dir, ) - return create_app(service) + # Explicit non-loopback mode retains the documented trusted-lab boundary. + # Default local mode rejects arbitrary Host headers (including DNS rebinding). + if config.bind_is_loopback: + return create_app(service) + return create_app(service, trusted_hosts=("*",)) def serve(config: WebConfig) -> int: diff --git a/src/variaq/web/static/app.css b/src/variaq/web/static/app.css index d28d024..e2833e8 100644 --- a/src/variaq/web/static/app.css +++ b/src/variaq/web/static/app.css @@ -1,31 +1,11 @@ /* VariaQ local web UI — technical, research-oriented, high-density. */ :root { - --bg: #f7f7f5; - --surface: #ffffff; - --ink: #1c1d20; - --muted: #5d6168; - --line: #d9dbde; - --accent: #14548c; - --accent-ink: #ffffff; - --ok: #1b6b3a; - --warn: #8c5a00; - --err: #9c1f1f; + color-scheme: dark; + --bg: #0b111c; --surface: #111c2c; --ink: #e9f0fc; --muted: #a8b8cf; + --line: #2a3b51; --accent: #89ccff; --accent-ink: #081828; + --ok: #85e0b4; --warn: #ffd18a; --err: #ff9baf; --mono: ui-monospace, "SF Mono", "Cascadia Mono", Consolas, monospace; - --radius: 6px; -} -@media (prefers-color-scheme: dark) { - :root { - --bg: #141518; - --surface: #1d1f24; - --ink: #e6e7ea; - --muted: #9aa0a8; - --line: #383b42; - --accent: #6ea8dd; - --accent-ink: #0e1a26; - --ok: #5aa776; - --warn: #d8a13e; - --err: #e07a7a; - } + --radius: 12px; } * { box-sizing: border-box; } html { -webkit-text-size-adjust: 100%; } @@ -177,3 +157,134 @@ legend { color: var(--muted); font-size: 0.85rem; padding: 0 0.4rem; } .analysis-groups { margin-top: 0.8rem; } .analysis-groups h3 { margin: 1rem 0 0.3rem; } + +/* Scientific workspace: shared visual rhythm across every page. */ +[hidden] { display: none !important; } +body { background: radial-gradient(ellipse at 90% 0, #142b43 0, transparent 45%), var(--bg); } +.topbar { background: #0d1624; } +.topbar-inner { max-width: 1400px; min-height: 80px; gap: 28px; padding: 16px 28px; } +.brand { display: flex; align-items: center; gap: 10px; font-size: 21px; letter-spacing: -.5px; } +.brand small { display: none; } +.brand-symbol { padding: 8px; border: 1px solid #49799c; border-radius: 10px; font: 700 19px var(--mono); color: var(--accent); } +.brand-symbol span { color: var(--ok); } +.nav { margin-left: auto; gap: 5px; } +.nav a { color: var(--muted); padding: 9px 12px; font-size: 14px; } +.nav a[aria-current=page] { background: #20394f; color: #b6e2ff; } +.container { max-width: 1400px; padding: 36px 28px 64px; } +.page-head { margin-bottom: 28px; } +.page-head h1 { font-size: clamp(26px, 3vw, 38px); letter-spacing: -1px; font-weight: 650; } +.page-head p { max-width: 740px; margin: 10px 0 0; } +.eyebrow { color: var(--ok); font-size: 11px; letter-spacing: .16em; text-transform: uppercase; font-weight: 700; margin-bottom: 10px; } +.page-head-row { flex-wrap: wrap; align-items: center; } +.panel { padding: 24px; margin-bottom: 24px; background: linear-gradient(140deg, #152234, #101b2b); } +.panel h2 { font-size: 18px; font-weight: 600; margin-bottom: 16px; letter-spacing: -.3px; } +.stat-grid { gap: 16px; margin-bottom: 28px; } +.stat { padding: 22px; background: linear-gradient(140deg, #182b3e, #111c2c); } +.stat-value { font-size: 34px; font-weight: 550; letter-spacing: -1px; } +.stat-label { margin-top: 8px; text-transform: none; letter-spacing: 0; font-size: 13px; } +.data-table th, .data-table td { padding: 12px 10px; } +.data-table th { text-transform: none; letter-spacing: 0; font-size: 12px; white-space: normal; } +.data-table tr:hover td { background: #17283a; } +.link-list { gap: 14px; } +.link-list li { padding: 8px 0; border-bottom: 1px solid var(--line); } +.status-success, .status-failed, .status-unavailable { display: inline-block; padding: 3px 8px; border-radius: 6px; background: #182a3a; font-size: 12px; } +button, .button { padding: 11px 18px; border-color: #426788; font-size: 14px; } +.button.primary, button[type=submit] { background: #b0deff; color: #0b1c2b; border-color: #b0deff; } +input, select, textarea { padding: 10px 12px; background: #0c1625; min-height: 42px; max-width: 100%; } +input[type=checkbox] { min-height: auto; accent-color: var(--accent); width: 17px; height: 17px; flex-shrink: 0; } +label { gap: 7px !important; } +fieldset { padding: 22px; } +legend { color: var(--ink); font-size: 15px; } +.actions, .pager { flex-wrap: wrap; } +.filters > label { flex: 1 1 145px; min-width: 0; } +.campaign-form input { width: 100%; } +.campaign-form input[type=checkbox] { width: 17px; } +.solver-card { padding: 14px; border: 1px solid var(--line); border-radius: 10px; background: #0d1828; align-items: start !important; flex: 1 1 220px; } +.solver-card small { display: block; margin-top: 4px; font-size: 12px; color: var(--muted); } +.solver-card:has(input:checked) { border-color: #70b4dc; background: #183047; } +.solver-card:has(input:disabled) { opacity: .7; } +.form-help { font-size: 12px; color: var(--muted); margin: 6px 0 16px; } +.workflow-steps { display: flex; flex-wrap: wrap; gap: 14px; padding: 0; list-style: none; margin: 0 0 24px; color: var(--muted); font-size: 13px; } +.workflow-steps li { padding: 8px 14px; border: 1px solid var(--line); border-radius: 30px; } +.workflow-steps li:first-child { color: var(--accent); border-color: #507d9d; } +summary { cursor: pointer; padding: 8px 0; color: var(--accent); font-weight: 550; } +details { margin-top: 14px; } +pre { white-space: pre-wrap; overflow-wrap: anywhere; } +.sr-data { overflow-wrap: anywhere; table-layout: fixed; } +.sr-data th, .sr-data td { padding: 8px 5px; } +.chart-box { min-width: 0; padding: 18px; background: #0e1928; border: 1px solid var(--line); border-radius: 12px; } +.chart-box figcaption { font-size: 14px; color: var(--ink); font-weight: 600; margin-bottom: 12px; } +.chart-note { font-size: 12px; color: var(--muted); } +.notice { overflow-wrap: anywhere; } +.notice.warn, .empty-state { padding: 18px; } +.tip { position: relative; display: inline-block; } +.tip button { padding: 2px 7px; font-size: 12px; border-radius: 50%; margin-left: 4px; } +.tip span { display: none; position: absolute; z-index: 20; bottom: 100%; left: 0; width: min(270px, 70vw); padding: 12px; background: #253c52; color: var(--ink); border: 1px solid #7396b4; border-radius: 8px; font-size: 13px; font-weight: normal; } +.tip:focus-within span, .tip:hover span { display: block; } +.sr-only { position: absolute; width: 1px; height: 1px; padding: 0; overflow: hidden; clip: rect(0,0,0,0); white-space: nowrap; } +@media (max-width: 800px) { + .topbar-inner { padding: 16px; gap: 12px; } + .nav { margin-left: 0; width: 100%; } + .nav a { padding: 8px 10px; } + .container { padding: 24px 16px 48px; } + .panel { padding: 18px; } + .stat-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; } + .stat { padding: 18px; } + .page-head-row { display: block; } + .page-head-row .actions { margin-top: 18px; } + .filters { gap: 12px; } + .grid-2col > *, .table-wrap { min-width: 0; max-width: 100%; } +} +@media (prefers-reduced-motion: reduce) { * { scroll-behavior: auto !important; transition: none !important; } } + +#comparison-repeats .data-table, #comparison-pairs .data-table, #comparison-runs .data-table { min-width: 0; table-layout: fixed; } +#comparison-repeats td, #comparison-pairs td, #comparison-runs td { white-space: normal; overflow-wrap: anywhere; vertical-align: top; } +#compare-ids { display: block; width: 100%; margin-top: 8px; } +.tip.dismissed span { display: none !important; } +@media (max-width: 600px) { + #comparison-repeats table, #comparison-pairs table, #comparison-runs table, + #comparison-repeats tbody, #comparison-pairs tbody, #comparison-runs tbody, + #comparison-repeats tr, #comparison-pairs tr, #comparison-runs tr, + #comparison-repeats td, #comparison-pairs td, #comparison-runs td { display: block; width: 100%; } + #comparison-repeats thead, #comparison-pairs thead, #comparison-runs thead { display: none; } + #comparison-repeats tr, #comparison-pairs tr, #comparison-runs tr { border: 1px solid var(--line); border-radius: 10px; margin-bottom: 12px; } + #comparison-repeats td::before, #comparison-pairs td::before, #comparison-runs td::before { content: attr(data-label); display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; } +} + +#comparison-workspace .chart-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } +.chart-legend { list-style: none; padding: 0; margin: 10px 0; font-size: 12px; color: var(--muted); } +.chart-legend li { display: flex; gap: 8px; margin: 6px 0; overflow-wrap: anywhere; } +.chart-legend li span { flex-shrink: 0; width: 16px; font-size: 16px; } +@media (max-width: 800px) { #comparison-workspace .chart-grid { grid-template-columns: 1fr; } } + +/* Focused comparison summaries and short, wrapping chart keys. */ +#comparison-summary { margin: 18px 0; } +#comparison-summary .stat { padding: 16px; } +#comparison-summary .stat-value { font-size: clamp(23px, 2.6vw, 32px); overflow-wrap: anywhere; } +.stat-detail { display: block; color: var(--muted); font-size: 11px; margin-top: 7px; } +.methodology { margin: 16px 0 22px; } +.methodology p { color: var(--muted); max-width: 85ch; margin: 12px 0; } +.solver-summary { margin-top: 22px; } +.group-context { display: block; color: var(--muted); font-size: 11px; margin: 5px 0; } +#comparison-workspace .chart-legend { display: flex; flex-wrap: wrap; gap: 4px 16px; } +#comparison-workspace .chart-legend li { margin: 3px 0; } +#comparison-gap-state { font-size: 12px; line-height: 1.6; } +#comparison-gap-state[data-state=all_zero] { border-left-color: var(--ok); } +#comparison-groups table { table-layout: fixed; min-width: 0; } +#comparison-groups td { white-space: normal; vertical-align: top; } +@media (max-width: 600px) { + #comparison-workspace .panel { padding: 16px; } + #comparison-workspace .chart-box { padding: 12px; } + #comparison-summary { gap: 8px; } + #comparison-summary .stat { padding: 12px; } + #comparison-groups table, #comparison-groups tbody, #comparison-groups tr, #comparison-groups td { display: block; width: 100%; } + #comparison-groups thead { display: none; } + #comparison-groups tr { border: 1px solid var(--line); border-radius: 10px; margin: 10px 0; padding: 10px; } + #comparison-groups td { border: 0; padding: 5px 0; } + #comparison-groups td:not(:first-child) { display: flex; justify-content: space-between; gap: 12px; } + #comparison-groups td:not(:first-child)::before { content: attr(data-label); color: var(--muted); font-size: 12px; } +} +@media (max-width: 600px) { + #comparison-groups td:nth-child(4) { flex-wrap: wrap; } + #comparison-groups td:nth-child(4) .group-context { width: 100%; margin: 0; text-align: right; } +} diff --git a/src/variaq/web/static/app.js b/src/variaq/web/static/app.js index d1bfa23..5349f53 100644 --- a/src/variaq/web/static/app.js +++ b/src/variaq/web/static/app.js @@ -25,7 +25,8 @@ async function handle(res) { const detail = payload && payload.error ? payload.error : payload && payload.detail ? (typeof payload.detail === "string" ? { message: payload.detail } : payload.detail) : { type: "HttpError", message: "Request failed with status " + res.status }; - const err = new Error(detail.message || "request failed"); + const message = Array.isArray(detail) ? detail.map(d => `${(d.loc || []).filter(x=>x!=="body").join(" → ")}: ${d.msg}`).join("; ") : detail.message; + const err = new Error(message || "Request failed. Check your selection and try again."); err.type = detail.type || "HttpError"; err.details = detail.details || null; throw err; @@ -117,13 +118,14 @@ async function initDashboard() { ${esc(r.solver_name)}${statusPill(r.status)} ${fmtNum(r.objective)}${fmtNum(r.wall_time_seconds, 4)}`), 5, "No runs yet."); + if (el("stat-completed")) el("stat-completed").textContent = data.fully_recorded_campaigns ?? "—"; el("recent-campaigns-list").innerHTML = data.recent_campaigns.length - ? data.recent_campaigns.map(c => `
  • ${esc(shortId(c.campaign_id))} ${esc(c.name)} (${esc(c.family)})
    • `).join("") + ? data.recent_campaigns.map(c => `
  • ${esc(shortId(c.campaign_id))} ${esc(c.name)} ${esc(c.family)} · ${c.completed_runs ?? "—"} recorded / ${c.requested_runs ?? "—"} requested Compare →
    • `).join("") : `
  • No campaigns yet.
    • `; const solvers = Object.entries(data.solver_availability); tableRows("solver-state-table", solvers.map(([name, s]) => - `${esc(name)}${boolText(s.installed)}${boolText(s.available)}`), 3, "No solvers."); + `${esc(name)} · ${name==='exact'||name==='heuristic'?'CPU classical':name==='cudaq-gpu'?'GPU local quantum simulator':'CPU local quantum simulator'}${boolText(s.installed)}${boolText(s.available)}`), 3, "No solvers."); const c = data.cudaq || {}; const targets = c.targets || {}; kvFill(el("backend-facts"), [ @@ -249,6 +251,49 @@ async function initRunDetail() { el("run-status").hidden = true; el("run-detail").hidden = false; const r = run.result, m = r.backend.metrics || {}; + el("run-configuration").textContent = JSON.stringify(run.solver_config, null, 2); + el("run-environment").textContent = JSON.stringify(run.environment, null, 2); + el("run-canonical").textContent = JSON.stringify(run.problem, null, 2); + kvFill(el("run-scope-kv"), [ + ["Complete-call wall scope", m.wall_time_scope || "Unknown in historical metadata"], + ["Compute scope", m.solver_time_scope || "Unknown in historical metadata"], + ["Requested precision", run.solver_config.parameters.precision || "Not requested"], + ["Cold-start verified", boolText(run.environment.benchmark_protocol?.cold_start_verified)], + ["Cold-start reason", run.environment.benchmark_protocol?.cold_start_reason || "No process/device isolation recorded"], + ]); + // Use the authoritative Python layer, with explicit local reference selection. + const evidence = await apiGet(`/runs?problem_id=${encodeURIComponent(run.problem.problem_id)}&limit=500`); + const ids = [...new Set([id,...evidence.runs.map(x=>x.run_id)])].slice(0,500); + const measured = await apiPost('/comparison',{run_ids:ids,limit:500}); + const instance = measured.analysis.instances.find(i=>i.problem_id===run.problem.problem_id); + const measurement = instance?.runs.find(r=>r.run_id===id); + if(measurement) { + kvFill(el("run-authoritative-kv"),[ + ["Authoritative decoded objective", fmtNum(measurement.objective)], + ["Authoritative feasibility", boolText(measurement.feasible)], + ["Reported objective agrees", boolText(measurement.objective_matches_reported)], + ["Selected exact reference", fmtNum(measurement.reference_optimum)], + ["Gap to selected exact reference (%)", fmtNum(measurement.optimality_gap_percent)], + ["Reference evidence", `${ids.length} selected problem runs (bounded at 500); exact sources: ${instance.reference_run_ids.join(', ') || 'none'}`], + ]); + el("run-missing").textContent=JSON.stringify(measurement.missing_measurements,null,2); + } + const reproduce = el("reproduce-run"); + reproduce.disabled = !run.reproduction_eligible; + reproduce.addEventListener("click",async()=>{ + reproduce.disabled=true; setStatus("reproduce-status","Starting a new local run with the saved configuration…"); + try { + const data=await apiPost(`/runs/${encodeURIComponent(id)}/reproduce`,{}); + async function poll() { + try { const {task}=await apiGet('/campaigns/tasks/'+encodeURIComponent(data.task.task_id)); + if(task.status==='completed') { el("reproduce-status").innerHTML=`Open reproduced experiment →`;reproduce.disabled=false; } + else if(['failed','interrupted'].includes(task.status)){setStatus('reproduce-status',task.error?.message || 'Reproduction failed',true);reproduce.disabled=false;} + else {setStatus('reproduce-status','Reproducing locally…');setTimeout(poll,1500);} + } catch(e){setStatus('reproduce-status','Task status unavailable. Check Runs for persisted results before retrying.',true);reproduce.disabled=false;} + } + poll(); + }catch(e){setStatus('reproduce-status',e.message,true);reproduce.disabled=false;} + }); const messages = el("run-messages"); const warnings = (r.warnings || []).slice(0, 10); const errors = (r.errors || []).slice(0, 10); diff --git a/src/variaq/web/static/app2.js b/src/variaq/web/static/app2.js index 4dac44e..be3b107 100644 --- a/src/variaq/web/static/app2.js +++ b/src/variaq/web/static/app2.js @@ -5,7 +5,7 @@ (function () { const { apiGet, apiPost, esc, html, fmtNum, fmtTime, fmtBytes, shortId, statusPill, boolText, setStatus, kvFill, tableRows } = window.VARIAQ; -const CHART_COLORS = ["#14548c", "#1b6b3a", "#8c5a00", "#9c1f1f", "#5b3f8c", "#0f6e73", "#745c00", "#333"]; +const CHART_COLORS = ["#89ccff", "#85e0b4", "#ffd18a", "#ff9baf", "#d0adff", "#98dbec"]; function groupKeyLabel(key) { return Object.entries(key).map(([k, v]) => `${k}=${v}`).join(", ") || "(all runs)"; @@ -117,7 +117,7 @@ async function initCampaigns() { ${c.requested_runs}${c.completed_runs}${esc(counts)}`; }), 7, "No campaigns yet. Create one from “New campaign”."); } catch (err) { - setStatus("campaigns-status", `${err.type}: ${err.message}`, true); + setStatus("campaigns-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -128,7 +128,7 @@ async function pollActiveTask(regionId) { let payload; try { payload = await apiGet("/campaigns/tasks/current"); } catch { return; } const task = payload.task; - if (!task || (task.status !== "starting" && task.status !== "running")) { + if (!task || task.status === "completed") { region.hidden = true; return; } @@ -139,8 +139,10 @@ async function pollActiveTask(regionId) { Campaign ${esc(task.status)} — ${task.completed_runs}${total ? " / " + total : ""} runs recorded - task ${esc(task.task_id)}`; - setTimeout(refresh, 2000); + task ${esc(task.task_id)} + ${task.error ? `

    ${esc(task.error.message)}

    ` : ""} + ${task.campaign_id ? `Inspect stored campaign` : ""}`; + if (["starting", "running"].includes(task.status)) setTimeout(refresh, 2000); } refresh(); } @@ -150,7 +152,7 @@ async function initCampaignDetail() { const id = decodeURIComponent(location.pathname.split("/").pop()); const runState = { offset: 0, limit: 25 }; try { - const data = await apiGet("/campaigns/" + encodeURIComponent(id)); + const data = await apiGet("/campaigns/" + encodeURIComponent(id) + "?limit=25"); el("campaign-status").hidden = true; el("campaign-detail").hidden = false; const d = data.definition; @@ -167,7 +169,7 @@ async function initCampaignDetail() { ]); kvFill(el("campaign-exec-kv"), [ ["Requested runs", data.requested_runs], - ["Completed runs", data.completed_runs], + ["Recorded runs (all outcomes)", data.completed_runs], ["Status counts", Object.entries(data.status_counts || {}).map(([k, v]) => `${k}: ${v}`).join(", ") || "—"], ["Solver breakdown", Object.entries(data.solver_breakdown || {}).map(([k, v]) => `${k}: ${v}`).join(", ") || "—"], ]); @@ -188,7 +190,7 @@ async function initCampaignDetail() { await generateCampaignReport(id, e.target); }); } catch (err) { - setStatus("campaign-status", `${err.type}: ${err.message}`, true); + setStatus("campaign-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -206,7 +208,7 @@ async function loadCampaignRuns(campaignId, state) { el("campaign-runs-prev").disabled = data.offset === 0; el("campaign-runs-next").disabled = data.offset + data.limit >= data.total; } catch (err) { - setStatus("campaign-status", `${err.type}: ${err.message}`, true); + setStatus("campaign-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -234,7 +236,7 @@ async function runAnalysis(campaignId) { drawScalingChart("chart-qubits", "chart-qubits-table", pts, "qubits", p => p.resource.qubits); } catch (err) { - el("analysis-body").innerHTML = `

    ${esc(err.type)}: ${esc(err.message)}

    `; + el("analysis-body").innerHTML = `

    ${esc(err.type || "Validation")}: ${esc(err.message)}

    `; } } @@ -267,7 +269,7 @@ async function generateCampaignReport(campaignId, form) { setStatus("report-gen-status", `Report ${result.report_id} written.${warn}`); await loadCampaignReports(campaignId); } catch (err) { - setStatus("report-gen-status", `${err.type}: ${err.message}`, true); + setStatus("report-gen-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -293,17 +295,19 @@ async function initCampaignNew() { const form = el("campaign-form"); let lastPlan = null; - // Solver picker from live capabilities (no hardcoded matrix). - try { - const caps = await apiGet("/capabilities"); - el("solver-picker").innerHTML = caps.solvers.map(s => - `` - ).join(""); - } catch (err) { - el("solver-picker").innerHTML = `${esc(err.message)}`; + let capabilities = []; + function renderSolvers() { + const family = el("campaign-family").value; + const selected = [...form.querySelectorAll('input[name="solver"]:checked')].map(c=>c.value); + el("solver-picker").innerHTML = capabilities.map(s => { + const compatible = (s.supported_families || []).includes(family); + const enabled = s.available && compatible; + const description = s.name === "exact" ? "CPU · proven reference, exponential enumeration" : s.name === "heuristic" ? "CPU · seeded classical search" : s.name === "cudaq-gpu" ? "GPU · local QAOA simulation" : "CPU · local QAOA simulation"; + return ``; + }).join(""); } + try { capabilities = (await apiGet("/capabilities")).solvers; renderSolvers(); } + catch (err) { el("solver-picker").innerHTML = `${esc(err.message)}`; } const familySelect = el("campaign-family"); function renderFamilyParams() { @@ -312,12 +316,19 @@ async function initCampaignNew() { `` ).join("") || `

    No additional parameters for this family.

    `; } - familySelect.addEventListener("change", renderFamilyParams); + familySelect.addEventListener("change", () => { renderFamilyParams(); renderSolvers(); invalidate(); }); renderFamilyParams(); function gatherDefinition() { - const sizes = form.problem_sizes.value.split(",").map(s => parseInt(s.trim(), 10)).filter(n => !Number.isNaN(n)); - const seeds = form.problem_seeds.value.split(",").map(s => parseInt(s.trim(), 10)).filter(n => !Number.isNaN(n)); + function integers(raw, positive=false) { + const parts=raw.split(',').map(s=>s.trim()); + if(parts.some(s=>! /^-?\d+$/.test(s) || !Number.isSafeInteger(Number(s)) || (positive && Number(s)<1))) throw new Error('Enter complete comma-separated integers; sizes must be positive.'); + const values=parts.map(Number); + if(new Set(values).size !== values.length) throw new Error('Remove duplicate sizes or seeds.'); + return values; + } + const sizes = integers(form.problem_sizes.value,true); + const seeds = integers(form.problem_seeds.value); const solvers = [...form.querySelectorAll('input[name="solver"]:checked')].map(c => c.value); const genParams = {}; for (const input of form.querySelectorAll("[data-gen-param]")) { @@ -332,6 +343,7 @@ async function initCampaignNew() { problem_sizes: sizes, problem_seeds: seeds, solvers, + solver_config: Object.fromEntries(solvers.filter(s=>['qaoa','cudaq-cpu','cudaq-gpu'].includes(s)).map(s=>[s,{parameters:{p:Number(el('qaoa-depth').value),optimizer_trials:Number(el('qaoa-trials').value),shots:Number(el('qaoa-shots').value),warmup:el('qaoa-warmup').checked,...(s.startsWith('cudaq')?{precision:s==='cudaq-gpu'?el('qaoa-precision').value:'fp64'}:{})}}])), repeats: parseInt(form.repeats.value, 10) || 1, base_seed: parseInt(form.base_seed.value, 10) || 0, generator_parameters: genParams, @@ -340,18 +352,37 @@ async function initCampaignNew() { }; } + function invalidate() { + lastPlan = null; + el("plan-section").hidden = true; + el("run-campaign-btn").disabled = true; + try { const d = gatherDefinition(); el("run-count-preview").textContent = `${d.problem_sizes.length * d.problem_seeds.length * d.solvers.length * d.repeats} requested runs · local sequential execution. Review the plan for exact enumeration and simulator memory estimates.`; } + catch { el("run-count-preview").textContent = 'Complete the problem grid to preview the run count.'; } + } + form.addEventListener("input", invalidate); + form.addEventListener("change", invalidate); + el("reference-recipe").addEventListener("change", e=>{ + if(e.target.value==='small') {form.problem_sizes.value='6';form.problem_seeds.value='11';} + if(e.target.value==='matrix') {form.problem_sizes.value='4, 6, 8';form.problem_seeds.value='11, 42';} + invalidate(); + }); + invalidate(); form.addEventListener("submit", async e => { e.preventDefault(); + lastPlan = null; + el("run-campaign-btn").disabled = true; + setStatus("run-status", ""); try { const definition = gatherDefinition(); const plan = await apiPost("/campaigns/plan", { definition }); + if(JSON.stringify(definition)!==JSON.stringify(gatherDefinition())) throw new Error("Setup changed while planning. Review the updated plan before starting."); lastPlan = { definition, plan }; renderPlan(plan); el("plan-section").hidden = false; - el("plan-section").scrollIntoView({ behavior: "smooth", block: "start" }); + el("plan-section").scrollIntoView({ behavior: "auto", block: "start" }); } catch (err) { el("plan-section").hidden = false; - el("plan-body").innerHTML = `

    ${esc(err.type)}: ${esc(err.message)}

    `; + el("plan-body").innerHTML = `

    ${esc(err.type || "Validation")}: ${esc(err.message)}

    `; } }); @@ -368,6 +399,7 @@ async function initCampaignNew() { default maximum ${plan.default_max_runs}

    ${rows}
    SolverRunsSupportedInstalledAvailable
    +
    Full reviewed definition & resource estimates
    ${esc(JSON.stringify(plan, null, 2))}
    ${unavailable ? `

    Unavailable combinations

    ` : ""} ${warnings ? `

    Warnings

    ` : ""}`; const over = plan.exceeds_default_max; @@ -390,6 +422,7 @@ async function initCampaignNew() { if (!lastPlan) return; const btn = el("run-campaign-btn"); btn.disabled = true; + form.querySelectorAll("input,select,textarea,button").forEach(n=>n.disabled=true); setStatus("run-status", "Starting…"); try { const over = lastPlan.plan.exceeds_default_max; @@ -398,10 +431,12 @@ async function initCampaignNew() { override_max_runs: over && el("override-max-runs").checked, }); setStatus("run-status", `Task ${result.task.task_id} ${result.task.status}.`); + sessionStorage.setItem("variaq-active-task", result.task.task_id); pollTaskUntilDone(result.task.task_id); } catch (err) { - setStatus("run-status", `${err.type}: ${err.message}`, true); + setStatus("run-status", `${err.type || "Validation"}: ${err.message}`, true); btn.disabled = false; + form.querySelectorAll("input,select,textarea,button").forEach(n=>n.disabled=false); renderSolvers(); } }); @@ -410,23 +445,38 @@ async function initCampaignNew() { const payload = await apiGet("/campaigns/tasks/" + encodeURIComponent(taskId)); const task = payload.task; const total = task.planned_runs; - if (task.status === "completed" || task.status === "failed") { + el("run-progress").hidden = false; + el("run-progress").max = total || 1; + el("run-progress").value = task.completed_runs; + if (["completed", "failed", "interrupted"].includes(task.status)) { + sessionStorage.removeItem("variaq-active-task"); + if (el("plan-body").textContent === 'Recovering tracked task status…') { + el("plan-body").textContent = 'Stored execution status.'; + } + form.querySelectorAll("input,select,textarea,button").forEach(n=>n.disabled=false); renderSolvers(); if (task.status === "completed") { const cid = task.campaign_id; - setStatus("run-status", `Completed: ${task.completed_runs} runs recorded.`); + setStatus("run-status", `Execution finished: ${task.completed_runs} runs recorded. Outcomes: ${JSON.stringify(task.result?.status_summary || {})}; skipped: ${task.result?.skipped_runs ?? "see campaign"}.`); el("plan-body").insertAdjacentHTML("beforeend", `

    Open campaign ${esc(shortId(cid))}

    `); } else { - setStatus("run-status", `Failed: ${task.error ? esc(task.error.message) : "unknown error"}`, true); + setStatus("run-status", `${task.status}: ${task.completed_runs} runs preserved. ${task.error ? task.error.message : "unknown error"}`, true); + if (task.campaign_id) { + el("plan-body").insertAdjacentHTML("beforeend", + `

    Inspect preserved campaign

    `); + } } return; } setStatus("run-status", `Running… ${task.completed_runs}${total ? " / " + total : ""} runs`); setTimeout(() => pollTaskUntilDone(taskId), 1500); } catch { - setStatus("run-status", "Lost contact with campaign task (server restarted?). The campaign runs already recorded remain in the database.", true); + setStatus("run-status", "Task status unavailable. Refresh to reconnect to stored task status. Completed records remain in Campaigns; duplicate submissions do not restart execution.", true); + form.querySelectorAll("input,select,textarea,button").forEach(n=>n.disabled=false); renderSolvers(); } } + const previousTask = sessionStorage.getItem("variaq-active-task"); + if(previousTask) { el("plan-section").hidden=false; el("plan-body").textContent='Recovering tracked task status…'; pollTaskUntilDone(previousTask); } } /* -- reports ------------------------------------------------------------------ */ @@ -444,7 +494,7 @@ async function initReports() { ${r.warnings && r.warnings.length ? `${r.warnings.length}` : "0"}`), 6, "No reports."); } catch (err) { - setStatus("reports-status", `${err.type}: ${err.message}`, true); + setStatus("reports-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -469,7 +519,7 @@ async function initReportDetail() { Download`), 4, "No files for this report."); } catch (err) { - setStatus("report-status", `${err.type}: ${err.message}`, true); + setStatus("report-status", `${err.type || "Validation"}: ${err.message}`, true); } } @@ -506,7 +556,7 @@ async function initCapabilities() { el("cap-warnings").innerHTML = data.warnings.map(w => `
  • ${esc(w.message)}
    • `).join(""); } } catch (err) { - setStatus("capabilities-status", `${err.type}: ${err.message}`, true); + setStatus("capabilities-status", `${err.type || "Validation"}: ${err.message}`, true); } } diff --git a/src/variaq/web/static/benchmark.js b/src/variaq/web/static/benchmark.js new file mode 100644 index 0000000..bde42af --- /dev/null +++ b/src/variaq/web/static/benchmark.js @@ -0,0 +1,161 @@ +/* Presentation only. Scientific evaluation, pairing and statistics come from Python. */ +"use strict"; +(function () { +const {apiGet, apiPost, esc, fmtNum, shortId, boolText, setStatus} = window.VARIAQ; +const $ = id => document.getElementById(id); +const palette = ['#89ccff', '#85e0b4', '#ffd18a', '#d0adff', '#ff9baf', '#98dbec']; +const charts = new Map(); +const label = r => `${r.solver} · ${r.backend.name} · ${r.precision || 'precision unreported'} · ${r.warmup_mode} · seed ${r.seed} · ${shortId(r.run_id)}`; +const json = value => `
    ${esc(JSON.stringify(value, null, 2))}
    `; +function table(headers, rows) { + return `
    ${headers.map(x=>``).join('')}${rows.length ? rows.map(row=>`${row.map((x,i)=>``).join('')}`).join('') : ``}
    ${esc(x)}
    ${x}
    No observations in this selection.
    `; +} +const metric = (v, unit='') => v == null ? 'Unavailable' : `${new Intl.NumberFormat('en', {maximumSignificantDigits:4}).format(v)}${unit}`; +const solverName = (solver, backend) => ({'exact/python-enumeration':'Exact · CPU','heuristic/seeded-multistart-local-search':'Heuristic · CPU','qaoa/qiskit.quantum_info.Statevector':'QAOA · Qiskit','cudaq-cpu/qpp-cpu':'CUDA-Q · CPU','cudaq-gpu/nvidia':'CUDA-Q · GPU'}[`${solver}/${backend}`] || `${solver} · ${backend}`); +const modeName = mode => ({explicit_warmup:'warmup',unwarmed:'unwarmed',not_applicable:'classical',unknown:'warmup unknown'}[mode] || mode); +const zeroMarkers = {id:'measuredZero',afterDatasetsDraw(chart) { + chart.data.datasets.forEach((dataset,index)=>{ + const meta=chart.getDatasetMeta(index); + if(!meta.visible) return; + dataset.data.forEach((value,i)=>{ + if(value !== 0) return; + const bar=meta.data[i]; if(!bar) return; + const ctx=chart.ctx; ctx.save();ctx.fillStyle=dataset.borderColor; + ctx.beginPath();ctx.arc(bar.x,bar.y,4,0,Math.PI*2);ctx.fill(); + if(chart.canvas.id==='compare-chart-gap') {ctx.font='12px system-ui';ctx.fillStyle='#e9f0fc';ctx.fillText('0%',bar.x+9,bar.y+4);} + ctx.restore(); + }); + }); +}}; +function draw(id, labels, datasets, note, scatter=false, context=[]) { + charts.get(id)?.destroy(); charts.delete(id); + const canvas = $('compare-chart-'+id); + const available=datasets.length && datasets.some(d=>d.data.some(v=>scatter ? v.y != null : v != null)); + canvas.parentElement.hidden=!available; + if(!scatter) canvas.parentElement.style.height=Math.max(180,labels.length*(id==='timing'?62:48)+50)+'px'; + $('compare-chart-'+id+'-note').textContent = note + (available ? '' : ' No measured values in this selection.'); + const legendId='compare-chart-'+id+'-legend'; + let legend=$(legendId); + if(!legend){legend=document.createElement('ul');legend.id=legendId;legend.className='chart-legend';legend.setAttribute('aria-label',id+' chart series');canvas.parentElement.insertAdjacentElement('afterend',legend);} + legend.hidden=!available || (!scatter && datasets.length===1); + legend.innerHTML=datasets.map((d,i)=>`
  • ${esc(d.label)}
    • `).join(''); + const rows = datasets.flatMap(d => d.data.map((v,i) => [esc(d.label), esc(scatter ? `${v.instance} · ${JSON.stringify(v.parameters)}` : context[i] || labels[i].join(' · ')), metric(scatter ? v.y : v, id==='gap'?'%':''), scatter ? v.count ?? '—' : d.counts?.[i] ?? '—'])); + $('compare-chart-'+id+'-data').innerHTML = `${esc(note)}MetricSolver / sourceValuen${rows.map(r=>`${r.map(v=>`${v}`).join('')}`).join('')}`; + if (typeof Chart === 'undefined' || !available) return; + const valuesAxis={beginAtZero:true,ticks:{color:'#a8b8cf'},grid:{color:'#23364b'}}; + // A visible marker represents a measured zero; nulls remain absent. + if(id==='gap') valuesAxis.suggestedMax=1; + const chart = new Chart(canvas, {type: scatter ? 'scatter' : 'bar',plugins:scatter?[]:[zeroMarkers],data:{labels,datasets: datasets.map((d,i)=>({...d,backgroundColor:palette[i%palette.length],borderColor:palette[i%palette.length],pointStyle:['circle','triangle','rect','cross','star','rectRot'][i%6],pointRadius:5,showLine:false,maxBarThickness:18}))},options:{indexAxis:scatter?'x':'y',responsive:true,maintainAspectRatio:false,scales:scatter?{x:{ticks:{color:'#a8b8cf',maxRotation:0},title:{display:true,text:'Logical variables',color:'#a8b8cf'},grid:{color:'#23364b'}},y:valuesAxis}:{x:valuesAxis,y:{ticks:{color:'#e9f0fc',autoSkip:false,font:{size:11}},grid:{display:false}}},plugins:{legend:{display:false},tooltip:{callbacks:{title:items=>scatter ? items[0].raw.instance : context[items[0].dataIndex] || labels[items[0].dataIndex].join(' · ')}}}}}); + charts.set(id,chart); +} +async function comparison() { + let selection = null, loaded = null, busy = false; + let instanceId = null; + function fill(id, values, all) { + const node=$(id), previous=node.value; + node.innerHTML = (all ? `` : '') + [...new Set(values)].map(v=>``).join(''); + if ([...node.options].some(o=>o.value===previous)) node.value=previous; + } + function render() { + const analysis=loaded.analysis; + const instances=analysis.instances || []; + let instance=instances.find(i=>i.problem_id===instanceId) || instances[0]; + if (instance) { instanceId=instance.problem_id; $('compare-instance').value=instanceId; } + const runs=instance?.runs || []; + const pairs=(analysis.comparisons.find(c=>c.comparison_type==='paired_solvers')?.pairs || []).filter(p=>p.problem_id===instanceId); + const complete=pairs.filter(p=>p.left_run_id && p.right_run_id); + const view=loaded.instance_summaries.find(i=>i.problem_id===instanceId); + const groups=view?.groups || []; + const achievement=view?.optimum_achievement_rate == null ? 'Unavailable' : `${metric(view.optimum_achievement_rate*100)}%`; + $('comparison-summary').innerHTML = [ + ['Reference optimum',metric(instance?.reference_optimum),instance ? instance.sense+' · selected exact runs' : 'No instance selected'], + ['Valid quality runs',view?.valid_run_count ?? 0,`${view?.excluded_run_count ?? 0} excluded · this instance`], + ['Optimum achievement',achievement,`${view?.optimum_achieved_count ?? 0} / ${view?.reference_comparable_count ?? 0} reference-comparable runs`], + ['Mean gap',metric(view?.gap_percent.mean,'%'),`n = ${view?.gap_percent.count ?? 0} measurable gaps`] + ].map(([k,v,detail])=>`
    ${esc(v)}${esc(k)}${esc(detail)}
    `).join(''); + $('comparison-window').textContent=`${loaded.selected_count} selected runs · ${instances.length} instances · window ${loaded.window_count ? loaded.offset+1 : 0}–${loaded.offset+loaded.window_count} of ${loaded.total}.`; + $('compare-prev').disabled=loaded.offset===0; $('compare-next').disabled=loaded.offset+loaded.limit>=loaded.total; + $('comparison-title').textContent=instance ? `${instance.problem.family || instance.problem.problem_type} · ${shortId(instance.problem_id)} · ${instance.sense}` : 'No instance measurements'; + $('comparison-reference').textContent=instance ? (instance.reference_optimum == null ? `Exact reference unavailable: ${instance.reference_reason}.` : `Reference ${metric(instance.reference_optimum)} · ${instance.reference_run_ids.length} selected exact run(s).`) : 'No runs in this selection. Adjust the filters or choose another window.'; + $('comparison-scope').textContent=`This instance only · ${complete.length} recorded pairs · descriptive means by configuration, not paired speedups.`; + const chartGroups=groups.slice(0,12); + $('comparison-chart-limit').hidden=groups.length<=12; + $('comparison-chart-limit').textContent=`Quality, gap and timing charts show the first 12 of ${groups.length} configurations. Summary metrics and solver tables include all configurations; filter the evidence for a focused chart.`; + const labels=chartGroups.map(g=>[solverName(g.solver,g.backend.name),`${g.precision?.toUpperCase() || ''} ${modeName(g.warmup_mode)} · ${g.group_id.replace('config-','C')}`.trim()]); + const contexts=chartGroups.map(g=>`${g.solver} · ${g.backend.name} · ${g.group_id} · ${g.valid_run_count} valid runs; source IDs in solver summary`); + draw('quality',labels,[{label:'Objective',data:chartGroups.map(g=>g.objective.mean),counts:chartGroups.map(g=>g.objective.count)}],'Mean feasible objective · same instance.',false,contexts); + draw('gap',labels,[{label:'Gap (%)',data:chartGroups.map(g=>g.gap_percent.mean),counts:chartGroups.map(g=>g.gap_percent.count)}],'Mean gap · lower is better. Dots labelled 0% are measured zeros.',false,contexts); + const counts=view?.gap_counts || {zero:0,nonzero:0,unavailable:0}; + const state=$('comparison-gap-state'); + state.dataset.state=view?.gap_state || 'unavailable'; + state.textContent=counts.zero || counts.nonzero ? `Run gaps in this instance: ${counts.zero} zero · ${counts.nonzero} nonzero · ${counts.unavailable} unavailable run gaps${view.gap_state==='all_zero'?' — all measurable gaps are 0%.':''}` : `Gap unavailable for ${counts.unavailable} runs${instance?.reference_reason ? ': '+instance.reference_reason : '. No defined relative gap in this selection.'}`; + draw('timing',labels,[['wall_time_seconds','Wall'],['solver_time_seconds','Compute'],['warmup_seconds','Warmup']].map(([key,name])=>({label:name,data:chartGroups.map(g=>g.timing[key].mean),counts:chartGroups.map(g=>g.timing[key].count)})), 'Mean seconds · scopes retained per configuration; layers overlap.',false,contexts); + $('comparison-groups').innerHTML=table(['Solver / configuration','Valid runs','Mean objective','Mean gap','Optimum achieved'],groups.map(g=>[ + `${esc(solverName(g.solver,g.backend.name))}${esc(g.precision?.toUpperCase() || 'Precision unreported')} · ${esc(modeName(g.warmup_mode))} · ${esc(g.group_id.replace('config-','C'))}
    Controls & source runs${json(g)}
    `, + `${g.valid_run_count} valid / ${g.run_ids.length} selected`,metric(g.objective.mean),`${metric(g.gap_percent.mean,'%')}n = ${g.gap_percent.count} · ${g.gap_counts.unavailable} unavailable`,g.optimum_achievement_rate==null?'Unavailable':`${metric(g.optimum_achievement_rate*100)}% · ${g.optimum_achieved_count}/${g.reference_comparable_count}` + ])); + const series = new Map(); + for(const p of loaded.repeat_scaling || []) { + if(p.wall.mean == null) continue; + const key=`${p.solver} / ${p.backend} / ${p.warmup_mode} / ${JSON.stringify(p.parameters)}`; + if(!series.has(key))series.set(key,[]); + series.get(key).push({x:p.size,y:p.wall.mean,count:p.wall.count,instance:p.problem_id,parameters:p.parameters}); + } + draw('scaling',[],[...series].map(([name, data],i)=>({label:solverName(name.split(' / ')[0],name.split(' / ')[1])+` · ${modeName(name.split(' / ')[2])} · S${i+1}`,data})), 'Mean wall seconds by size · separate points for each instance and configuration.',true); + $('comparison-repeats').innerHTML=table(['Solver / backend / warmup','Status counts','Objective n / mean / median / sample SD / range','Wall s: n / mean / median / sample SD / range'],(instance?.repeats || []).map(r=>[`${esc(r.solver)} / ${esc(r.backend)} / ${esc(r.warmup_mode)}
    Configuration & source runs${json({parameters:r.parameters,run_ids:r.run_ids})}
    `,esc(JSON.stringify(r.status_counts)), summary(r.objective), summary(r.timing.wall_time_seconds)])); + $('comparison-pairs').innerHTML=table(['Repeat / benchmark','Solvers & status','Pairing','Expectation delta / reason'],pairs.map(p=>[`${esc(p.repeat_key)}
    ${esc(shortId(p.benchmark_id))}`,`${esc(p.left_solver)} (${esc(p.left?.status || 'missing')}) / ${esc(p.right_solver)} (${esc(p.right?.status || 'missing')})`,p.left_run_id && p.right_run_id ? 'Recorded pair' : `Incomplete / ambiguous: ${esc(JSON.stringify(p.missing_pairs))}`,`${fmtNum(p.numerical_correctness.max_expectation_delta)}
    ${esc(p.numerical_correctness.reason || 'Matched candidate expectations')}
    Pair provenance${json(p)}
    `])); + if(pairs.length) $('comparison-pairs').innerHTML = `
    Explore ${pairs.length} paired / incomplete observations${$('comparison-pairs').innerHTML}
    `; + if (!pairs.length) $('comparison-pairs').insertAdjacentHTML('afterbegin',runs.length ? '

    Unpaired evidence: no recorded repeat pairs in this instance. Individual measurements remain available.

    ' : '

    No recorded experiments match this selection. Adjust the filters or choose another campaign.

    '); + $('comparison-runs').innerHTML=table(['Experiment / backend','Status / feasibility','Objective / gap %','Wall / compute / warmup (s)','Unavailable reasons & provenance'],runs.map(r=>[`${esc(shortId(r.run_id))}
    ${esc(label(r))}`,`${esc(r.status)} / ${boolText(r.feasible)}`,`${metric(r.objective)} / ${metric(r.optimality_gap_percent)}`,`${metric(r.timing.wall_time_seconds)} / ${metric(r.timing.solver_time_seconds)} / ${metric(r.timing.warmup_seconds)}`,`
    ${Object.keys(r.missing_measurements).length} unavailable measurements${json(r.missing_measurements)}
    Full provenance & scopes${json(r)}
    `])); + setStatus('comparison-status', `${loaded.selected_count} runs analyzed. ${analysis.warnings.length ? analysis.warnings.length+' scientific warnings; see provenance.' : ''}`); + if (analysis.warnings.length) $('comparison-runs').insertAdjacentHTML('afterbegin',`
    ${analysis.warnings.map(esc).join('
    ')}
    `); + $('comparison-export').disabled=loaded.selected_count===0; + } + function summary(s) { return `${s.count} / ${fmtNum(s.mean,5)} / ${fmtNum(s.median,5)} / ${s.std==null?'Unavailable (fewer than 2 observations)':fmtNum(s.std,5)} / ${fmtNum(s.minimum,5)}–${fmtNum(s.maximum,5)}`; } + async function load(next, reset=true) { + if(busy) return; busy=true; $('comparison-export').disabled=true; + setStatus('comparison-status','Analyzing selected evidence…'); $('comparison-workspace').hidden=true; + try { + loaded=await apiPost('/comparison',next); selection=next; + const instances=loaded.analysis.instances || [], runs=instances.flatMap(i=>i.runs); + fill('compare-instance', instances.map(i=>i.problem_id)); + if(reset) { fill('compare-problem',instances.map(i=>i.problem_id),'All instances'); fill('compare-solver',runs.map(r=>r.solver),'All solvers'); fill('compare-backend',runs.map(r=>r.backend.name),'All backends'); fill('compare-size',loaded.analysis.scaling_points.map(p=>p.x_value),'All sizes'); } + render(); $('comparison-workspace').hidden=false; + } catch(e) { setStatus('comparison-status',e.message,true); } + finally { busy=false; } + } + $('comparison-form').addEventListener('submit',e=>{e.preventDefault(); const ids=$('compare-ids').value.trim(); const campaign=$('compare-campaign').value; if(!ids&&!campaign){setStatus('comparison-status','Choose a campaign or enter run IDs.',true);return;} instanceId=null; for(const id of ['compare-problem','compare-solver','compare-backend','compare-size','compare-warmup']) $(id).value=''; load(ids?{run_ids:ids.split(',').map(s=>s.trim()).filter(Boolean),limit:500,offset:0}:{campaign_id:campaign,limit:500,offset:0});}); + $('compare-instance').addEventListener('change',()=>{instanceId=$('compare-instance').value;render();}); + $('comparison-filters').addEventListener('submit',e=>{e.preventDefault();if(!selection)return;const next={...selection,offset:0}; for(const [key,id] of [['problem_id','compare-problem'],['solver','compare-solver'],['backend','compare-backend'],['warmup','compare-warmup'],['size','compare-size']]) {delete next[key];if($(id).value)next[key]=key==='size'?Number($(id).value):$(id).value;}load(next,false);}); + for(const [id,dir] of [['compare-prev',-1],['compare-next',1]]) $(id).addEventListener('click',()=>load({...selection,offset:Math.max(0,selection.offset+dir*selection.limit)},false)); + $('comparison-export').addEventListener('click',async()=>{if(!selection||busy)return; $('comparison-export').disabled=true;setStatus('comparison-export-status','Generating exports…');try{const result=await apiPost('/comparison/reports',selection);$('comparison-export-status').innerHTML=`Open report and download JSON, CSV, Markdown →`;}catch(e){setStatus('comparison-export-status',e.message,true);}finally{$('comparison-export').disabled=false;}}); + try { const data=await apiGet('/campaigns?limit=100'); $('compare-campaign').innerHTML=''+data.campaigns.map(c=>``).join(''); const cid=new URLSearchParams(location.search).get('campaign');if(cid){if(!data.campaigns.some(c=>c.campaign_id===cid))$('compare-campaign').insertAdjacentHTML('beforeend',``);$('compare-campaign').value=cid;await load({campaign_id:cid,limit:500,offset:0});}else setStatus('comparison-status',data.campaigns.length?'Choose a campaign to begin.':'No campaigns yet. Create an experiment or select existing run IDs.'); }catch(e){setStatus('comparison-status',e.message,true);} +} +if(window.VARIAQ_PAGE==='comparison') document.addEventListener('DOMContentLoaded',comparison); +// Dark, readable defaults also apply to historical campaign charts. +document.addEventListener('DOMContentLoaded',()=>{ + document.querySelectorAll('[data-help]').forEach((node,i)=>{ + node.className='tip'; const id='scientific-help-'+i; + node.innerHTML=`${esc(node.dataset.help)}`; + function show(){node.classList.remove('dismissed');const tip=node.querySelector('[role=tooltip]');tip.style.left='0';const box=tip.getBoundingClientRect();if(box.right>innerWidth-12)tip.style.left=(innerWidth-12-box.right)+'px';} + node.addEventListener('mouseenter',show);node.querySelector('button').addEventListener('focus',show); + node.addEventListener('keydown',e=>{if(e.key==='Escape')node.classList.add('dismissed');}); + }); + let pending = false; + function accessibleScrollRegions() { + if(pending) return; pending=true; + requestAnimationFrame(()=>{ + pending=false; + document.querySelectorAll('.table-wrap, .json-block').forEach(node=>{ + const scrollable=node.clientWidth>0 && (node.scrollWidth>node.clientWidth+1 || node.scrollHeight>node.clientHeight+1); + if(scrollable) {node.tabIndex=0;node.setAttribute('role','group');node.setAttribute('aria-label','Scrollable data. Use arrow keys to scroll.');} + else {node.removeAttribute('tabindex');node.removeAttribute('role');node.removeAttribute('aria-label');} + }); + }); + } + new MutationObserver(accessibleScrollRegions).observe(document.body,{childList:true,subtree:true}); + window.addEventListener('resize',accessibleScrollRegions); + document.addEventListener('toggle',accessibleScrollRegions,true); + accessibleScrollRegions(); + if(typeof Chart!=='undefined'){Chart.defaults.color='#a8b8cf';Chart.defaults.borderColor='#2a3b51';}}); +})(); diff --git a/src/variaq/web/task_journal.py b/src/variaq/web/task_journal.py new file mode 100644 index 0000000..9e40b05 --- /dev/null +++ b/src/variaq/web/task_journal.py @@ -0,0 +1,137 @@ +"""Durable web task snapshots with one OS-owned local worker slot.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from variaq.campaigns.store import CampaignStore +from variaq.models import utc_now + + +class TaskJournal: + def __init__(self, store: CampaignStore) -> None: + self.store = store + self._worker_file = None + lock_path = Path(str(store.path.resolve()) + ".web-worker.lock") + worker_file = lock_path.open("a+b") + try: + if os.name == "nt": + import msvcrt + + worker_file.seek(0, 2) + if worker_file.tell() == 0: + worker_file.write(b"0") + worker_file.flush() + worker_file.seek(0) + msvcrt.locking(worker_file.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(worker_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + worker_file.close() + else: + self._worker_file = worker_file + with store._connection() as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS web_tasks (" + "task_id TEXT PRIMARY KEY, request_key TEXT UNIQUE, record_json TEXT NOT NULL)" + ) + if self.can_execute: + self.reconcile() + + @property + def can_execute(self) -> bool: + return self._worker_file is not None + + def close(self) -> None: + if self._worker_file is not None: + self._worker_file.close() + self._worker_file = None + + def __del__(self) -> None: + self.close() + + def save(self, task: dict[str, Any], request_key: str | None = None) -> None: + with self.store._connection() as connection: + connection.execute( + "INSERT INTO web_tasks(task_id, request_key, record_json) VALUES (?, ?, ?) " + "ON CONFLICT(task_id) DO UPDATE SET record_json=excluded.record_json", + (task["task_id"], request_key, json.dumps(task)), + ) + + def get(self, *, task_id: str | None = None, request_key: str | None = None) -> dict | None: + key, value = ("task_id", task_id) if task_id else ("request_key", request_key) + with self.store._connection() as connection: + row = connection.execute( + f"SELECT record_json FROM web_tasks WHERE {key}=?", (value,) + ).fetchone() + return json.loads(row[0]) if row else None + + def current(self) -> dict | None: + with self.store._connection() as connection: + row = connection.execute( + "SELECT record_json FROM web_tasks ORDER BY rowid DESC LIMIT 1" + ).fetchone() + return json.loads(row[0]) if row else None + + def reconcile(self) -> None: + # Called only while owning the OS worker lock: no other web worker can + # still be executing. A killed process releases this lock automatically. + with self.store._connection() as connection: + rows = connection.execute( + "SELECT r.run_id, r.status, r.solver_name, r.problem_id, r.created_at, " + "r.record_json FROM runs r LEFT JOIN campaign_runs c ON c.run_id=r.run_id " + "WHERE c.run_id IS NULL" + ).fetchall() + for row in rows: + campaign_id = ( + json.loads(row["record_json"]).get("environment", {}).get("campaign_id") + ) + if ( + campaign_id + and connection.execute( + "SELECT 1 FROM campaigns WHERE campaign_id=?", (campaign_id,) + ).fetchone() + ): + connection.execute( + "INSERT INTO campaign_runs VALUES (?, ?, ?, ?, ?, ?)", + ( + campaign_id, + row["run_id"], + row["status"], + row["solver_name"], + row["problem_id"], + row["created_at"], + ), + ) + tasks = connection.execute("SELECT record_json FROM web_tasks").fetchall() + records = [json.loads(r[0]) for r in connection.execute("SELECT record_json FROM runs")] + for row in tasks: + task = json.loads(row[0]) + matching = [ + r + for r in records + if r.get("environment", {}).get("web_task_id") == task["task_id"] + ] + task["completed_runs"] = len(matching) + if task.get("source_run_id") and matching: + task["result"] = matching[0] + if task["status"] in {"starting", "running"}: + task["status"] = "interrupted" + task["finished_at"] = utc_now() + task["error"] = { + "type": "Interrupted", + "message": "The server stopped before recording a final task state. " + "Completed " + "records are preserved. Remaining work was not retried; an in-flight " + "result may not have been saved. Inspect the campaign before explicitly " + "planning a new campaign with a different name.", + } + connection.execute( + "UPDATE web_tasks SET record_json=? WHERE task_id=?", + (json.dumps(task), task["task_id"]), + ) diff --git a/src/variaq/web/tasks.py b/src/variaq/web/tasks.py index bb845cd..c9bd6c6 100644 --- a/src/variaq/web/tasks.py +++ b/src/variaq/web/tasks.py @@ -1,10 +1,10 @@ """In-process campaign execution task tracking for the local web UI. -0.7 campaign execution is local and sequential — one campaign at a time, -owned by a daemon thread inside the server process. This deliberately avoids -Redis/Celery/queue infrastructure. The task record is in-memory only: if the -server restarts mid-campaign, recorded runs remain in storage (append-only), -but the task status is lost and reported as unknown. +Campaign execution is local and sequential — one campaign at a time, +owned by a daemon thread inside the server process. Task snapshots are retained +in SQLite, while experiment records remain append-only. After a restart, unfinished tasks +are marked interrupted without retrying execution. An OS-held file lock prevents +multiple web workers from executing against the same database. """ from __future__ import annotations @@ -16,6 +16,7 @@ from typing import Any from variaq.models import utc_now +from variaq.web.task_journal import TaskJournal log = logging.getLogger("variaq.web.tasks") @@ -23,7 +24,7 @@ @dataclass class CampaignTask: task_id: str - status: str = "starting" # starting | running | completed | failed + status: str = "starting" # starting | running | completed | failed | interrupted definition: dict[str, Any] = field(default_factory=dict) override_max_runs: bool = False max_runs: int = 500 @@ -34,6 +35,8 @@ class CampaignTask: campaign_id: str | None = None error: dict[str, str] | None = None result: dict[str, Any] | None = None + kind: str = "campaign" + source_run_id: str | None = None def to_dict(self) -> dict[str, Any]: return { @@ -46,6 +49,8 @@ def to_dict(self) -> dict[str, Any]: "campaign_id": self.campaign_id, "error": self.error, "result": self.result, + "kind": self.kind, + "source_run_id": self.source_run_id, } @@ -56,6 +61,30 @@ def __init__(self) -> None: self._lock = threading.Lock() self._current: CampaignTask | None = None self._thread: threading.Thread | None = None + self._journal: TaskJournal | None = None + + def bind(self, service: Any) -> None: + if self._journal is None: + self._journal = TaskJournal(service.campaign_store) + + def close(self) -> None: + """Release the worker slot only when no native call can still be running.""" + with self._lock: + if (self._current is not None and self._current.status in {"starting", "running"}) or ( + self._thread is not None and self._thread.is_alive() + ): + log.warning("Worker still running; retaining execution lock until process exit") + return + if self._journal is not None: + self._journal.close() + + def _save(self, task: CampaignTask, key: str | None = None) -> None: + if self._journal is not None: + self._journal.save(task.to_dict(), key) + + @staticmethod + def _restore(data: dict | None) -> CampaignTask | None: + return CampaignTask(**data) if data else None def start( self, @@ -67,9 +96,29 @@ def start( planned_runs: int | None, ) -> CampaignTask | None: """Begin a campaign task, or return None if one is already active.""" - from variaq.campaigns.model import ExperimentCampaign + from variaq.campaigns.model import ExperimentCampaign, campaign_definition_id + from variaq.errors import ValidationError + + campaign = ExperimentCampaign.from_dict(definition) + campaign_id = campaign_definition_id(campaign) with self._lock: + if self._journal is not None: + existing = self._journal.get(request_key=campaign_id) + if existing: + return self._restore(existing) + if not self._journal.can_execute: + return None + try: + service.campaign_store.get_campaign(campaign_id) + except ValidationError: + pass + else: + raise ValidationError( + "This campaign already exists with historical execution state. Inspect it " + "before planning a new campaign with a different name; " + "it cannot be resumed." + ) if self._current is not None and self._current.status in {"starting", "running"}: return None task = CampaignTask( @@ -78,16 +127,20 @@ def start( override_max_runs=override_max_runs, max_runs=max_runs, planned_runs=planned_runs, + campaign_id=campaign_id, ) + self._save(task, campaign_id) self._current = task def work() -> None: with self._lock: task.status = "running" + self._save(task) def on_run_recorded(_run: Any, completed: int) -> None: with self._lock: task.completed_runs = completed + self._save(task) try: planned = ExperimentCampaign.from_dict(definition).requested_runs @@ -98,6 +151,7 @@ def on_run_recorded(_run: Any, completed: int) -> None: override_max_runs=override_max_runs, max_runs=max_runs, on_run_recorded=on_run_recorded, + web_task_id=task.task_id, ) with self._lock: task.status = "completed" @@ -112,6 +166,7 @@ def on_run_recorded(_run: Any, completed: int) -> None: finally: with self._lock: task.finished_at = utc_now() + self._save(task) self._thread = threading.Thread(target=work, name="variaq-campaign", daemon=True) self._thread.start() @@ -119,10 +174,64 @@ def on_run_recorded(_run: Any, completed: int) -> None: def current(self) -> CampaignTask | None: with self._lock: + if self._journal is not None: + return self._restore(self._journal.current()) return self._current + def start_reproduction(self, service: Any, run_id: str) -> CampaignTask | None: + """Reproduction shares the one local execution slot with campaigns.""" + original = service.get_run(run_id) + if not original["reproduction_eligible"]: + from variaq.errors import ValidationError + + raise ValidationError("Only local experiments can be reproduced") + with self._lock: + key = "reproduction:" + run_id + if self._journal is not None: + existing = self._journal.get(request_key=key) + if existing: + return self._restore(existing) + if not self._journal.can_execute: + return None + if self._current and self._current.status in {"starting", "running"}: + return None + task = CampaignTask( + task_id=f"task-{uuid.uuid4().hex[:12]}", + planned_runs=1, + kind="reproduction", + source_run_id=run_id, + ) + self._save(task, key) + self._current = task + + def work() -> None: + with self._lock: + task.status = "running" + self._save(task) + try: + result = service.reproduce(run_id, web_task_id=task.task_id) + with self._lock: + task.result = result + task.completed_runs = 1 + task.status = "completed" + except Exception as exc: + log.exception("Reproduction task %s failed", task.task_id) + with self._lock: + task.status = "failed" + task.error = {"type": type(exc).__name__, "message": str(exc)} + finally: + with self._lock: + task.finished_at = utc_now() + self._save(task) + + self._thread = threading.Thread(target=work, name="variaq-reproduce", daemon=True) + self._thread.start() + return task + def get(self, task_id: str) -> CampaignTask | None: with self._lock: + if self._journal is not None: + return self._restore(self._journal.get(task_id=task_id)) if self._current is not None and self._current.task_id == task_id: return self._current return None diff --git a/src/variaq/web/templates/base.html b/src/variaq/web/templates/base.html index fae31d3..6d9e3da 100644 --- a/src/variaq/web/templates/base.html +++ b/src/variaq/web/templates/base.html @@ -3,7 +3,7 @@ - + {% block title %}VariaQ{% endblock %} @@ -12,9 +12,10 @@
    - VariaQ + VariaQ RESEARCH LAB