diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 79cc18b..b31ad71 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,7 +1,7 @@ name: Publish to PyPI # Runs only when a GitHub Release is published (not on push or PR). -# Release flow: bump `version` in pyproject.toml and src/verdi/__init__.py, +# Release flow: bump `version` in pyproject.toml and src/verdi_agents/__init__.py, # merge, then publish a release tagged v (e.g. v0.1.1). on: release: @@ -21,8 +21,8 @@ jobs: run: | tag="${GITHUB_REF_NAME#v}" pkg=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") - mod=$(python -c "import verdi; print(verdi.__version__)") - echo "tag=$tag pyproject=$pkg verdi.__version__=$mod" + mod=$(python -c "import verdi_agents; print(verdi_agents.__version__)") + echo "tag=$tag pyproject=$pkg verdi_agents.__version__=$mod" if [ "$tag" != "$pkg" ] || [ "$tag" != "$mod" ]; then echo "::error::Release tag v$tag does not match pyproject ($pkg) / __version__ ($mod)" exit 1 diff --git a/.gitignore b/.gitignore index 99de71c..327c01e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ artifacts/ # GitHub Pages build output (site/build.py) _site/ + +# Local working notes (not published) +notes/ diff --git a/AUDIT.md b/AUDIT.md deleted file mode 100644 index 529f6f6..0000000 --- a/AUDIT.md +++ /dev/null @@ -1,176 +0,0 @@ -# System Audit and Remediation — verdi - -Started 2026-11-03 on branch `ci/pypi-release` (base `4664c96`). All changes are -uncommitted working-tree edits; nothing was pushed or published. - -## Verification commands - -| Check | Command | Baseline | Final | -|---|---|---|---| -| Lint (CI gate) | `ruff check .` | clean | clean | -| Tests (CI gate) | `python -m pytest` | 672 passed, 1 skipped | 769 passed, 1 skipped | -| Types (not a CI gate) | `mypy src/verdi --ignore-missing-imports` | 73 errors | 66 errors (all triaged, see R7) | -| Packaging | `pip wheel . --no-deps` + fresh venv, `verdi --help`, `verdi init`, `verdi sweep … --dry-run` | worked | not re-run (no packaging config changed) | -| Formatting | `ruff format --check .` | 156 files would change | unchanged — not enforced by CI; mass reformat deliberately skipped | - -Secret scan of tracked files and history (`sk-…`, `AKIA…`, `Bearer …`): only -obvious test fixtures (`tests/audit/test_judge_live.py`, `tests/experiments/test_g5_close.py`). -`.env` is git-ignored and was never committed. - -## Method - -Pass 1: four independent read-only reviewers (world/governance/schemas/scheduler; -society/agents/pilot/config; analysis/measurement/audit; CLI/experiments/packaging/docs) -plus mypy triage. Every finding was re-checked against the code before fixing. -Pass 2: an independent reviewer over the diff (regressions) and a fresh reviewer over -areas pass 1 only skimmed (pilot, g5_*, paired_run, measurement, adapters, recorded). -Each fix has a regression test that was confirmed to FAIL on the pre-fix source -(`git stash push -- src/…`) and pass after. - -New tests live in `tests/*/test_audit_regressions.py` (agents, world, governance, -society, analysis, experiments). `tests/test_cli.py` was updated (see F22). - -## Fixed findings (most severe first) - -| # | Sev | Where | Defect | Fix | -|---|---|---|---|---| -| F1 | Critical | `world/reducer.py` `_transfer`, `world/validate.py` | Transfer to self minted resources (recipient read before sender debit written back): `food=10`, `transfer{to:self,units:10}` → 20. Validator accepted it, so an LLM could do it every tick. | Validator rejects with new non-sanctionable code `self_transfer`; reducer raises `InvariantViolationError` as defence in depth. | -| F2 | High | `governance/constitution.py` | Two amendments enacted in one assembly: second was built from the shared submission base, silently dropping the first's rules. | New version = delta applied onto `active_at(effective_tick)`; vote rules still from the submission base. | -| F3 | High | `governance/constitution.py` | Caller-supplied votes counted raw: no dedup per voter, wrong-proposal votes counted, ballots `cast_vote` had rejected still counted; ineligible voters decided the threshold. | Recorded ballot is the single source of truth; caller votes go through `record_vote`; tally uses the same eligibility filter as quorum. | -| F4 | High | `governance/constitution.py` | Out-of-range `quorum`/`amendment_threshold` in a payload: amendment recorded as valid, then `ConstitutionVersion` raised, leaving store inconsistent. | Validated as fractions in (0, 1] → `invalid_quorum` / `invalid_amendment_threshold`. | -| F5 | High | `governance/sanctions.py`, `appeals.py`, `society/campaign.py` | One revocation per (agent, right): expiring/overturning one of two overlapping sanctions restored the right while the other was active; campaign cleared `under_sanction` on any expiry. | `RightsView` keeps all intervals; `SanctionEngine.end()` re-revokes from remaining live sanctions; campaign keeps the flag while `has_live_sanction`. | -| F6 | High | `agents/client.py` | 200 response with non-JSON body raised raw `JSONDecodeError` (not `ModelError`), aborting whole campaign batches; `"content": null` propagated `None` → `TypeError`/`AttributeError` in policy parsing, reflection, consolidation. | Parse inside the guarded block; `None` → `""`; non-string content → `ModelError`. | -| F7 | High | `analysis/comparative_report.py` | Enactment row used `f"…".format()` after inserting rule values: dict values crashed the report (`KeyError`), `{v}` in values was substituted, values double-escaped. | Single f-string, each value escaped once. | -| F8 | High | `analysis/outcome_metrics.py` `recovery_time` | Declared due ticks zipped with outcome events in YAML order, but the campaign evaluates in due-tick order; shipped `examples/full.yaml` declares `[12, 30, 46, 30]` → outcomes mislabelled. Manifest + resolved YAML ticks were also appended twice, forcing the raw-tick fallback; YAML errors crashed. | Resolved YAML replaces manifest ticks; ticks sorted (stable) before zip; YAML errors handled. | -| F9 | High | `experiments/sweep.py` | Ctrl-C: `with ThreadPoolExecutor` drained the queue, *starting* every queued run (spending model calls), then marked finished runs "interrupted". | Manual executor; `shutdown(cancel_futures=True)` on interrupt; finished-but-unrecorded results kept. | -| F10 | High | `experiments/sweep.py` | Resume reused "complete" runs by `run_id` only (a changed `--ticks` was ignored); runs from earlier invocations leaked into results, summary and exit code. | Per-run `scenario_sha256`; mismatch reruns; `current_runs` filters `SweepResult`, `collect()` and the exit code. Legacy entries without a hash are trusted as before. | -| F11 | High | `experiments/interview_experiment.py` | Post-pass audit always crashed: `asyncio.run()` given the dict returned by synchronous `audit_from_file`. | Call it directly. | -| F12 | Medium | `interview_experiment.py` | Post-pass audit ignored the scenario `model:` block (wrong/missing endpoint). | Uses `resolve_endpoint_config(load_scenario(...))`. | -| F13 | Medium | `config.py`, `g5_pairs.py`, `society/live.py` | Leak guards only checked `MODEL_API_KEY`/`VERDI_VLLM_API_KEY`; a key read via `model.api_key_env` was never scanned for. `ModelEndpointConfig.__repr__` printed the key. | Every resolved key is tracked for `api_key_values()`; G5 artifact audit scans all keys; `api_key` is `repr=False`. | -| F14 | Medium | `config.py` | `max_retries < 0` made every call raise `AssertionError("unreachable")`; zero timeout / concurrency accepted. | Range-checked with readable `ValueError`. | -| F15 | Medium | `society/campaign.py`, `memory.py` | Consolidation prompt's "existing summary" contained every recent entry (string split on the rendered history). | `AgentMemory.consolidated_summary()`. Note: changes consolidation prompt text (see R4). | -| F16 | Medium | `society/tasks.py`, `campaign.py` | One bad field (`"amount": null`) discarded the whole orchestrated tape; invalid transfer targets produced tasks the validator must reject; failed-parse calls were never logged, so token accounting undercounted. | Per-field fallback; invalid transfer → produce; meta returned with `parse_error` and recorded. | -| F17 | Medium | `society/live.py`, `agents/client.py` | Stall watchdog: reflection/consolidation write `model_calls.jsonl` only after the whole batch, so a long healthy batch could trip `os._exit(3)`; retry window ignored pool/connect timeouts and backoff. | In-process `completed_calls` counter feeds the watchdog; window includes pool + connect + max backoff per attempt. | -| F18 | Medium | `world/branch.py` | Branching after a prefix ending in a rejected event always failed (`after_hash=None`). | Terminal hash = `after_hash or before_hash`. | -| F19 | Medium | `world/policy.py`, `validate.py` | `max_produce_units_per_tick` was a per-action cap without an economy block. | Cumulative using `produced_this_tick` when supplied. | -| F20 | Medium | `measurement/longitudinal.py` `_parse_choice` | `"Yes."`, `'"yes"'`, `"Yes, I would."` scored as the low pole; Likert took the first digit anywhere (`"0 to 5 … 4"` → 0, `"10"` → 1, `"7 out of 10"` → crash of the whole probe round). | First-word match for word options; last standalone in-range integer for Likert; anything else flagged unparsed instead of raising. | -| F21 | Medium | `experiments/paired_run.py` | Missing `output_root` → `TypeError`; `except BaseException` swallowed Ctrl-C and continued; resume ignored scenario changes; `--concurrency` help promised `VERDI_MAX_CONCURRENCY` but code hard-coded 4. | Readable error; `except Exception`; per-arm `scenario_sha256`; default honours the env var. | -| F22 | Medium | `cli.py` | `verdi run x.yaml --run-dir=out` ignored the `=` form; bare `verdi` printed a pointer to a non-existent `docs/IMPLEMENTATION_PLAN.md`. | Both forms detected; bare `verdi` prints help (exit 0 kept for compatibility). Test updated to assert the help text instead of the stale string. | -| F23 | Medium | `pilot/driver.py` | Sanction evidence could cite a quarantine rejection of the subject's proposal (false sanction); default live capture dir had 1-second resolution (two runs could share one `live_tape.jsonl`). | Evidence only from the scripted violation; dir stamp has microseconds + pid. Offline canonical path unchanged. | -| F24 | Low | `world/validate.py` | Non-string `resource` (list/dict/null from model JSON) crashed with `TypeError`. | Rejected as `missing_argument`. | -| F25 | Low | `world/snapshot.py` | `load` never re-hashed content; digest used unvalidated as a filename (`../`). | Digest must be 64 hex; content hash verified on load. | -| F26 | Low | `governance/constitution.py` | A proposal could be enacted twice; version names could collide with an existing one and be silently skipped. | `ProposalNotInVoteError` on re-enact; unique version names. | -| F27 | Low | `world/economy.py` | `role_consumption` for a resource not in `consumption_per_tick` was silently ignored. | Rejected with a clear config error (shipped examples unaffected). | -| F28 | Low | `audit/live_bridge.py` | Scorer `ValueError` recorded as score 0.0 (worst), indistinguishable from judge disagreement. | `None` + `rubric_score_error`. | -| F29 | Low | `experiments/sweep.py` | Dot-only labels (`..`) escaped the sweep folder; bad sweep YAML / `vary: null` gave tracebacks; setup I/O errors aborted the sweep. | Labels sanitised; readable `SweepError`s; setup inside the per-run guard. | -| F30 | Low | `analysis/cross_run_compare.py` | `"tick": null` in provenance crashed (`int(None)`). | Non-numeric ticks skipped. | -| F31 | Medium | `experiments/sweep.py` (regression from F10, found by pass-2 diff review; also pre-existing for failed-run retries) | Relaunching a run into its old directory appended a second ledger to `events.jsonl` / `model_calls.jsonl` (two `event_id: 0`, broken hash chain). | Previous non-empty attempt is renamed to `.prev-` (never deleted) before relaunch. | -| F32 | Low | `analysis/outcome_metrics.py` (regression from F8) | Malformed `due_tick` (e.g. `soon`) raised instead of degrading, in both the resolved-YAML and manifest paths. | Falls back to raw ledger ticks as before. | -| F33 | High | `society/campaign.py` (found pass 3) | Production cap is documented per agent per tick (`economy.py`, `examples/full.yaml`, `institutional_recovery_constrained.yaml`), but the campaign summed production across ALL agents: once one agent hit the cap, every later producer that tick was rejected with `law:production_limit`. Also completes F19 (the cumulative constitutional cap was never fed in runs without an economy block). | `produced_this_tick` keyed by actor, tracked with or without an economy block. Campaign-level test in `tests/society/test_economy_campaign.py`. **Changes outcomes of economy runs vs earlier results.** | -| F34 | High | `audit/judge_live.py` `KEY_MATERIAL_PATTERNS` (found pass 3) | Leak guard matched ordinary speech (`Bearer of bad news`, `Authorization: the assembly approved…`): one such interview answer aborted `_write_artifacts` mid-way (truncated `interviews.jsonl`, all later artifacts lost) and judge tape writes. The guard's exception also echoed the matched secret into logs/findings. | Bearer / api_key / authorization patterns require a token-shaped value (all real key shapes still flagged, see tests); messages show only a 6-char prefix + length. `tests/experiments/test_g5_close.py::test_b2_planted_key_material_fails_leakage` updated to assert the finding locates the leak and does **not** contain the key (stricter, not weaker). | -| F35 | Medium | `measurement/interview_values.py` `_nearest_event` (pass 3) | Returned the agent's EARLIEST event, and compared lifecycle ticks (0–29) with ledger transition ticks (1–373): every trajectory row linked to the agent's first action (checked on `runs/interview-12agent-seed42`). | Lifecycle tick from the event's task id (`soc-tNN-`), carried forward for events without one; latest match wins. | -| F36 | Medium | `experiments/g5_close.py` (pass 3) | One judge rationale containing "causally" made the G5-close report unbuildable (frozen causal-word guard ran over verbatim model text). | Guard runs on the report with model-authored `rationale` / `raw` fields blanked. | -| F37 | Low–Med | `audit/interview_audit.py` (pass 3) | SimpleAudit reports scenario/judge failures as `severity="ERROR"` without raising; rows had `error: None`, so summaries reported 0 errors when the endpoint was down. | `severity == ERROR` sets `error`. | -| F38 | Medium | `measurement/longitudinal.py` (regression in F20, pass 3) | First F20 version rejected `"4."`/`"4.0"` and picked the scale top from `"4/5"`, `"4 out of 5"`, `"(scale 0-5)"`. | Scale ranges and denominators removed before taking the last standalone integer. | -| F39 | Medium | `experiments/paired_run.py` (pass 3) | Re-running a failed/stale arm appended into the old run dir (same root cause as F31). | Shares `_set_aside_previous_attempt`. | -| F40 | High | `analysis/outcome_metrics.py` `exclusion_episodes` (pass 4) | Open episodes ran to the last LEDGER tick (transition counter) while sanction ticks are LIFECYCLE ticks: `runs/interview-6agent-god-orch` grace sanctioned at t19 of 20 got `excluded_ticks: 85`. Overlapping sanctions were summed (dave 7 instead of 6) and flagged as repeat episodes. | Run end from `resources.jsonl` (ledger tick only as fallback); intervals merged per agent; `repeat_episode` = more than one merged interval. Now grace 0, dave 6. | -| F41 | Medium | `society/campaign.py` `_compute_metrics`, `live.py` (pass 4) | `metrics.json` `sanctions` counted world shocks and rejected impositions; `appeals` counted filed + adjudicated rows (6 for 3 appeals). Flows into sweep `summary.csv`. | Count accepted `sanction_imposed` and `phase == filed` only. | -| F42 | Medium | `society/prompting.py` (pass 4) | Action prompt said "It is now tick 104" in a 20-tick run (ledger counter), while durations/deadlines/caps count lifecycle ticks. | Uses `task.tick_created`; `SOCIAL_PROMPT_VERSION` bumped to `/5` (prompt text changes). | -| F43 | Low | `analysis/comparative_report.py` (pass 4) | Undefined baseline (no objective due in the preregistered window) was labelled "CENSORED — never recovered". | Labelled "not evaluable"; preregistered thresholds untouched (see R14). | -| F44 | Medium | `measurement/longitudinal.py` (regression in F38, pass 4) | "Last integer wins" let trailing reasoning override the answer (`"Choice: 4 … 2 factors"` → 2); JSON `{"choice": 4}` (int) fell through to free-text parsing. | First in-range standalone integer; int `choice` accepted. | -| F45 | Low–Med | `audit/judge_live.py` (regression in F34, pass 4) | Narrowed patterns missed `Authorization: Basic …`, digit-free keys, `"api_key": "…"` (JSON) and `x-api-key`. | Optional scheme word, 20+-char digit-free values, quote before separator, `x-api-key`. Zero hits over 398 real run artifacts + the plotly report bundle; ordinary speech still passes. | -| F46 | Low | `experiments/g5_close.py` (pass 4) | Quarantined rows' `error` embeds the model's stance verbatim, still able to trip the causal guard. | `error` added to excluded model-text keys. | -| F47 | Medium | `analysis/outcome_metrics.py` `completion_rate` (pass 5) | `due_tick` was the outcome event's LEDGER tick (`runs/smoke_final`: 56/127/198 for declared 10/24/38); shown in the comparative report and sweep/pair CSVs. | Declared due tick from `scenario_resolved.yaml` by objective id, else lifecycle tick carried from task ids; raw value kept as new `ledger_tick` field (schema test updated for the added field). | -| F48 | Medium | `analysis/outcome_metrics.py` `recovery_time` (exposed by F47) | A 0.0 pre-shock baseline gave threshold 0, so a run where nothing ever completed "recovered". `test_recovery_time_recovers_on_completed_post_delivery` only passed before because it fed the ledger tick as the due tick. | Zero baseline treated like an undefined one (not evaluable → censored). | -| F49 | Medium | `society/inner_life.py` (pass 5; site F42 missed) | Inner-life prompt also said "It is now tick "; enabled in both shipped full scenarios. | Lifecycle tick passed from the worker; `INNER_LIFE_PROMPT_VERSION` → `/2`. | -| F50 | Low–Med | `society/campaign.py` `_RecordingCaller` (pass 5) | Probe rows in `model_calls.jsonl` got their subject in COMPLETION order (parallel calls): `runs/smoke_final` had 14/24 probe rows attributed to the wrong agent (tokens/latency, HTML agent filter). | Subject taken before the await (send order). | -| F51 | Low | `audit/judge_live.py` (pass 5, regression in F45) | Case-insensitive `Bearer` + 16 hyphenated chars flagged prose ("standard-bearer counter-proposals"). | Case-sensitive `Bearer` followed by a digit-bearing 16+ token. | -| F52 | Medium | `world/validate.py` (pass 6) | Two accepted `produce 1e308` pushed a balance to `inf`; next tick `int(inf)` in `society/tasks.py` raised `OverflowError` and aborted the run (any scenario without an economy, incl. `starter.yaml`). | Any action whose destination balance would become non-finite is rejected (non-sanctionable code). | -| F53 | Low | `world/validate.py` (pass 6) | `"amount": true` accepted as 1 unit. | Booleans are not quantities. | -| F54 | Medium | `world/economy.py` `parse_task_objectives` (pass 6) | Duplicate objective ids (incl. auto `obj-NN` colliding with an explicit id) overwrote another objective's baseline, flipping completed → failed. | Duplicate ids rejected at load. | -| F55 | Medium | `analysis/scenario.py` `build_t0_state` (pass 6) | Values that load fine then crash deep into a live run: `target_pool` typo (crash at the due tick, after 57 calls), negative / NaN t0 resources, non-integer shock ticks (silently skipped), NaN/inf shock deltas (NaN silently zeroed the resource), empty agent list. | Load-time `ScenarioLoadError`s. All shipped scenarios (examples/, experiments/, tests/, src/) still load. | -| F56 | Medium | `analysis/outcome_metrics.py` `completion_rate` (pass 6, refinement of F47) | Task-id carry-forward lagged a tick when every action of the due tick was quarantined. | Under an economy, lifecycle tick = end-of-tick `consumption` events so far − 1 (exact); task ids only as fallback. | -| F57 | Low | `analysis/comparative_report.py` (pass 6, completes F43/F48) | A 0.0 baseline still printed "CENSORED — never recovered". | Labelled "not evaluable — pre-shock baseline rate is 0". | - -## Remaining risks / not fixed (with justification) - -| # | Sev | Where | Issue | Why not fixed | -|---|---|---|---|---| -| R1 | High | `measurement/probes.py:128-137` | Value axis is `names[i % 5]` but template is `templates[i % 3]`: the "solidarity" item text says "This is about cooperation"; private items 3–4 repeat dilemmas under different axes. | Changes the measurement instrument. Frozen pilot tapes must replay byte-for-byte and existing runs would stop being comparable. Needs an owner decision and an `instrument_version` bump. | -| R2 | High | `measurement/probes.py:125`, `scoring.py:96` | Yes/no items always score yes=1.0, no=0.1 regardless of item polarity: "accept a secret side-deal? yes" scores integrity 1.0. | Same as R1: an instrument redesign (per-item `reverse_scored`) that changes historical scores. | -| R3 | Medium | `analysis/estimates.py:287, 377-389` | Undefined estimands become 0.0 (no actions → safety 0, no sanctions → false-punishment 0, all-zero Gini → 0); a missing pre-sanction level becomes 0.0, so the agent always "recovers". | The 0.0 convention is written into the preregistration addendum. Changing estimand definitions is a research decision; flag to the analysis owner. | -| R4 | Medium | `society/campaign.py` consolidation (F15) | Consolidation prompts now differ from before, so recorded/replayed runs that captured the old prompt text will not reproduce consolidation calls byte-for-byte. | Intended: the old prompt was wrong. Live runs are not byte-reproducible anyway. | -| R5 | Medium | `measurement/longitudinal.py` | Unparsed probe rows still enter drift calculations (they are flagged but not filtered). | Filtering changes the drift estimand; F20 makes unparsed rows much rarer. | -| R6 | Low | `world/replay.py:90-97`, `branch.py:206-220` | Generic `replay`/`branch_from_snapshot` fold every `TapeStep` as an action; campaign-only kinds (`rights_change`, `world_shock`, …) are not understood. The campaign has its own correct replay. | Only matters if someone feeds a campaign tape to the generic replay; no caller does today. A shared dispatcher is a refactor. | -| R7 | Low | mypy (66 errors) | All triaged: none found to be a runtime bug beyond those fixed. Mostly Optional narrowing and loose annotations. | mypy is not a CI gate; fixing annotations is churn with no behaviour change. | -| R8 | Low | `agents/pool.py`, `campaign.py:_run_bounded` | `asyncio.gather` without cancellation: one failing call leaves siblings running detached. | Moving to `TaskGroup` changes error semantics (ExceptionGroup); low impact since callers abort the run anyway. | -| R9 | Low | `society/memory.py` | `max_entries` is stored but never enforced (growth bounded only by consolidation, which the campaign always runs). | No path appends without consolidating today. | -| R10 | Low | `experiments/treatments.py:135-145` | `rights_restoration_treatment` reports `applied=True` with a tape identical to placebo. | Naming/semantics of a treatment definition; owner decision. | -| R11 | Low | `pilot/live.py:236` | ADR-010 call budget checked only after the run. | Pilot path is sequential and short; enforcing mid-run changes pilot semantics. | -| R12 | Low | `.github/workflows/*` | CI and release install with `pip install -e`, never testing the built wheel's `force-include` examples. | Workflow change best done by the maintainer; packaging was verified manually (wheel + fresh venv + `verdi init`). | -| R13 | Low | `ruff format` | 156 files not formatted. | Not enforced; reformatting would bury the substantive diff. | -| R14 | High | `analysis/comparative_report.py:199-205` | Preregistered recovery criterion (baseline window ticks 0–10, post-shock ≥14, sustained 3 ticks) cannot be evaluated for the shipped `institutional_recovery_constrained.yaml` (due ticks 12, 30, 30, 46): no baseline objective, and 3 consecutive objective-bearing ticks never occur. Every run is "not evaluable" (was mislabelled CENSORED, F43). | The thresholds are explicitly preregistered ("defined BEFORE reading results"); changing them is a research decision. Options: amend the preregistration, or change the scenario's due ticks. | -| R15 | Low | `examples/full.yaml:56-57`, `docs/SCENARIOS.md` | `sanction_policy.max_duration_ticks` and `appeal_policy.deadline_ticks` are inert in campaign runs (fixed 3-tick sanctions; deadline always set explicitly). | Either wire them into `campaign.py` (changes run outcomes) or document them as inert; owner decision. | -| R16 | Low | `analysis/outcome_metrics.py` `recovery_time` | Reports `censored=True` when the baseline is undefined (tests pin this). | Report layer now distinguishes it (F43); changing the metric's contract is an analysis decision. | -| R17 | Low | `analysis/scenario.py`, `campaign.py` | Shocks, assemblies and deliveries scheduled at or after `total_ticks` are silently ignored (a late delivery drops out of `completion_rate`'s denominator). | Not made an error: `verdi sweep --ticks N` deliberately shortens runs below declared ticks. A warning (or reporting dropped objectives in metrics) is the right follow-up. | -| R18 | Low | `analysis/outcome_metrics.py` `exclusion_episodes` | An episode still open at run end is counted up to, not including, the last lifecycle tick (an agent sanctioned only on the final tick shows 0). | Convention pinned by the existing `test_exclusion_repeat_episode_flag_synthetic`; owner decision. | -| R19 | Low | `measurement/longitudinal.py` `_parse_choice` | An answer given after reasoning ("Considering 2 factors, I choose 4") parses as 2. | Same as the original first-digit rule; the prompt asks for a single digit. A structured `{"choice": N}` reply is parsed exactly. | -| R20 | Low | `society/memory.py` | `AgentMemory.from_checkpoint` ignores the stored `max_entries`. | No effect today (R9; `from_checkpoint` unused, `checkpoint.json` write-only). | - -## Behaviour changes researchers should know about - -These fixes intentionally change results compared with runs made before the audit: - -- **F33** per-agent production caps: economy runs accept produces that were rejected before. -- **F42 / F49** prompt text: the action prompt is now `verdi.society-prompt/5` and the inner-life prompt is `verdi.inner-life/2`. They show the lifecycle tick. -- **F15** consolidation prompt text. -- **F16** orchestrator fallbacks. -- **F1 / F52 / F53** validator now rejects self-transfers, overflowing amounts and boolean amounts. -- **F3** ineligible ballots are no longer counted. -- **F2** stacked amendments now apply together. -- **F5** overlapping sanctions are kept separate. -- **Metrics:** F40 and F41 (`metrics.json` counts and `excluded_ticks`), and F47 and F48 (`due_tick`, zero-baseline recovery). - -Recorded tapes still replay, because replay re-applies the recorded validations. Comparisons with older runs should account for these changes. - -## Verification results (final) - -- `ruff check .`: clean. -- `python -m pytest`: **769 passed, 1 skipped** (baseline 672 passed, 1 skipped). Of the 97 new tests, almost all live in `tests/*/test_audit_regressions.py`, plus one campaign-level test in `tests/society/test_economy_campaign.py`. Each was confirmed to fail on the pre-fix source. -- **Existing tests changed (3):** - - `tests/test_cli.py`: the stale-message assertion now checks the help output. - - `tests/experiments/test_g5_close.py::test_b2_planted_key_material_fails_leakage`: now asserts the finding locates the leak **without** repeating the key. This is stricter. - - `tests/analysis/test_outcome_metrics.py::test_completion_rate_well_formed`: adds the new `ledger_tick` field to the pinned key set. - - No test was deleted or skipped, and no assertion was weakened. -- `mypy`: 66 errors (baseline 73). All were triaged as typing-only (R7). -- Every shipped scenario YAML still passes `build_t0_state`. -- The key-material patterns produce 0 hits over 398 real run artifacts plus the bundled plotly report. - -## Audit passes and convergence - -| Pass | Scope | New actionable findings | -|---|---|---| -| 1 | 4 area reviewers + mypy triage | ~30 (1 critical, 10 high) | -| 2 | Diff review + lightly covered areas | 2 regressions + 9 new (4 high, all measurement-instrument; R1/R2) | -| 3 | Diff review + full read of pattern-scanned files | 3 regressions + 4 new (1 high: F34) | -| 4 | Diff review + end-to-end data-flow sweep | 3 regressions + 6 new (2 high: F40, R14) | -| 5 | Diff review + tick-unit / metrics sweep | 2 low regressions + 3 medium | -| 6 | Diff review + edge-input / scenario-validation probing | 2 low/medium refinements + 5 medium/low (no high) | - -**The stopping criterion was not formally met.** That criterion is two consecutive independent passes with no new actionable findings, and pass 6 still found medium-severity input-validation issues. All of them were fixed in this session. - -Severity has fallen with each pass. No unresolved critical issue remains. The only unresolved high-severity items are R1, R2 and R14. Each is a measurement-instrument or preregistration decision that needs the research owner and is documented above with evidence. - -## Continuation instructions - -1. Decide R1, R2 and R14 with the research owner: the probe-bank axis and polarity, and the preregistered recovery window versus the shipped scenario. Bump `instrument_version` if the probe bank changes. -2. Run two more independent passes. Suggested angles that have not been tried yet: - - The adapters (`adapters/concordia*.py`) under real Concordia objects. - - `html_report.py` with large runs (size and performance). - - Concurrency soak (`tests/agents/test_client_soak.py`) at higher fan-out. - - The g5 pipeline end to end on a real `runs/` root. -3. Optional hygiene, with no behaviour change: - - Fix the 66 mypy annotations and add mypy to CI. - - Make the PyPI workflow test the built wheel (R12). - - Do a separate `ruff format` commit (R13). -4. All work is uncommitted on branch `ci/pypi-release` (37 files changed). Split it into reviewable commits (world/governance, society, analysis/measurement, experiments/CLI, tests) on a fresh branch before opening a PR. diff --git a/README.md b/README.md index 4c9c769..dd855fc 100644 --- a/README.md +++ b/README.md @@ -6,204 +6,166 @@ [![CI](https://github.com/SimulaMet/Verdi/actions/workflows/ci.yml/badge.svg)](https://github.com/SimulaMet/Verdi/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/SimulaMet/Verdi/blob/main/LICENSE) -**An auditable testbed for value drift and recovery in societies of AI agents.** +**Simulate a small society of AI agents and watch how their values and rules +change over time.** -**Website and guides: https://simulamet.github.io/Verdi/** +You describe a village in one YAML file: who lives there, what they own, what +they say they care about, and what rules they live under. Verdi lets each +agent (a large language model playing a character) act step by step: produce +food, trade, take from the shared store, hoard, break rules, get punished, +appeal, and meet to vote on new rules. At the end you get one web page, +`report.html`, that shows what happened and why. -Verdi runs a society of LLM-driven agents over time and measures how their -declared and observed values change as they interact, negotiate shared rules, -and respond to institutional incentives. Every decision, law, sanction, appeal, -and state transition is written to an immutable event log, and every run renders -itself into a single self-contained `report.html` you open in a browser. +It is built for questions like: -What it models: -- Longitudinal, individual **and** collective behavior. -- Immutable universal rules plus a democratically amendable constitution. -- Social influence, adversarial behavior, sanctions, appeals, and rehabilitation. -- Reproducible, seeded experiments (same config + seed → comparable runs). -- Evidence-based auditing and transparent reporting via SimpleAudit. +- Do agents keep acting on the values they claim to hold, or drift away from them? +- When a crisis hits, does a society recover better if it is allowed to change its own rules? +- How does one selfish agent affect everyone else? +- Does the behavior depend on which language model you use? -**Status:** early but real. Experiments run against a real Qwen model and produce -auditable artifacts; findings are published only after validation. +``` + scenario.yaml ──► each step, every agent sees the world ──► proposes an action + │ + report.html ◄── everything is logged ◄── rules accept or reject it +``` --- ## Install -The easiest way is [uv](https://docs.astral.sh/uv/). Nothing to set up: -`uvx` fetches Verdi into a cached, isolated environment on first use. +You need Python 3.10+ and access to a language model that speaks the OpenAI +API (a local vLLM or Ollama server, or a hosted provider). ```bash -uvx verdi-agents init my-study # example scenarios + sweeps + a .env template -cd my-study # put your key in .env, your endpoint in examples/starter.yaml -uvx verdi-agents run examples/starter.yaml # one run -> runs/starter/report.html -uvx verdi-agents sweep examples/sweeps/model_comparison.yaml --dry-run +pip install verdi-agents ``` -Prefer a permanent command? Run `uv tool install verdi-agents` or -`pip install verdi-agents`, then use `verdi init`, `verdi run` and `verdi sweep`. -The package is `verdi-agents` on PyPI; the command and `import verdi` are `verdi`. - -To work on Verdi itself, clone the repo instead (next section). +One name everywhere: install `verdi-agents`, run `verdi-agents`, and in Python +`import verdi_agents` (Python names cannot contain a hyphen). -## Quickstart from source (60 seconds) +No install? [uv](https://docs.astral.sh/uv/) runs it directly: put `uvx` in +front of any command below, for example `uvx verdi-agents init my-study`. -For working on Verdi itself. Verdi calls any OpenAI-compatible endpoint. For -this example it is pointed at a Qwen model served by vLLM — but any compatible -endpoint works. +## Your first run ```bash -# 1. Get the code and install it -git clone https://github.com/SimulaMet/Verdi.git -cd Verdi -python -m venv .venv && source .venv/bin/activate -pip install -e . - -# 2. Configure the endpoint (never commit real values) -cp .env.example .env -# then edit .env and set MODEL_API_KEY; model name + base_url go in the scenario `model:` block - -# 3. Run the starter society (4 agents, 20 ticks, 1 constitutional meeting) -source .env -python -m verdi.society.live \ - --scenario examples/starter.yaml \ - --run-dir runs/my-run \ - --concurrency 4 +verdi-agents init my-study # example scenarios + a .env file for your key +cd my-study ``` -That's it. When it finishes, open the result: +1. Open `.env` and set `MODEL_API_KEY` (leave it empty if your server needs no key). +2. Open `examples/starter.yaml` and set `base_url` and `name` under `model:` to your model server. ```bash -open runs/my-run/report.html # macOS +verdi-agents run examples/starter.yaml +open runs/starter/report.html # macOS; on Linux use xdg-open ``` -`report.html` is a self-contained interactive page — the society, the social -network, resource and value timelines, governance, the sanction/appeal history, -and the constitution's evolution. The engine writes it automatically at the end -of every run (local, no extra model calls); pass `--no-report` to skip it. +The starter society has four agents (a farmer, a researcher, a guard and a +selfish hoarder), runs for 20 steps and holds one rule-making meeting. -### See it without running first +## Commands -A finished run of a larger 4-agent, 50-tick society (kept at `tests/fixtures/scenarios/minimal.yaml`) is committed at -[examples/minimal/society-minimal-report.html](https://github.com/SimulaMet/Verdi/blob/main/examples/minimal/society-minimal-report.html) -— open it in a browser. It shows one agent who starts **already under sanction**: -his rejected transfer attempts, the `law:under_sanction` code, his rights being -restored when the sanction expires, and constitutional meetings. +| Command | What it does | +|---------|--------------| +| `verdi-agents init [folder]` | Create a folder with example scenarios and a `.env` template | +| `verdi-agents run ` | Run one society; results go to `runs//` | +| `verdi-agents sweep ` | Rerun a scenario many times, changing only what you list (models, seeds, settings) | +| `verdi-agents pair ` | Run the same society twice: once with fixed rules, once where agents may change the rules | +| `verdi-agents compare ` | Measure and compare the two halves of a `pair` run | ---- - -## Where results live - -Every run writes one folder (`runs/my-run/` here). Open `report.html` first; the -rest is the raw, reconstructable evidence: - -| File | What it is | -|------|------------| -| `report.html` | Interactive summary — open this first | -| `run_meta.json` | Status, model, seed, replay check, wall time | -| `metrics.json` | The numbers (actions, rejections, sanctions, appeals) | -| `timeline.md` | Human-readable, tick-by-tick story | -| `events.jsonl` | The immutable event ledger (ground truth) | -| `model_calls.jsonl` | Every model call (replay provenance) | -| `sanctions.jsonl` | Sanctions, appeals, and restorations | -| `constitution_history.jsonl` | The constitution, version by version | -| `probes.jsonl` | Declared vs. observed values per probe phase | -| `resources.jsonl` | World and per-agent balances per tick | +Add `--help` to any command for its options. --- -## Scenarios - -Each scenario YAML is a self-contained, frozen input: model, agents, resources, -governance and schedule all live in one file. - -- [examples/starter.yaml](https://github.com/SimulaMet/Verdi/blob/main/examples/starter.yaml) — the smallest working scenario (~40 lines). Copy it and tweak. -- [examples/full.yaml](https://github.com/SimulaMet/Verdi/blob/main/examples/full.yaml) — every available knob, commented. Tested offline so it cannot drift. -- [examples/institutional_recovery_constrained.yaml](https://github.com/SimulaMet/Verdi/blob/main/examples/institutional_recovery_constrained.yaml) — the constrained-economy base for the paired study. - -Reference: [docs/SCENARIOS.md](https://github.com/SimulaMet/Verdi/blob/main/docs/SCENARIOS.md). +## Words used in Verdi -You only need to fill the **Required** section (plus each agent's fields); the -**Derivable** section is optional and the engine fills sensible defaults. JSON -works too — YAML is a superset. +| Word | Meaning | +|------|---------| +| **Scenario** | One YAML file describing the whole experiment. A run reads nothing else. | +| **Agent** | One member of the society, played by the language model with a name, role, personality and starting belongings. | +| **Tick** | One time step. Every agent gets a chance to act each tick. | +| **Commons** | The shared store of food and credits that everyone can draw from. | +| **Declared values** | Five scores (integrity, fairness, cooperation, autonomy, solidarity) an agent starts with and says it holds. | +| **Universal law** | Rules that can never change (for example, nobody can own a negative amount). | +| **Constitution** | Rules the society can change, such as the most you can take from the commons at once. | +| **Assembly** | A scheduled meeting where agents propose and vote on changes to the constitution. | +| **Shock** | A planned crisis, for example half the food disappears at tick 10. | +| **Sanction / appeal** | A punishment that limits an agent's rights for a while, and the agent's chance to contest it. | +| **Value check** | Questions put to agents during a run to see whether what they say and what they do still match their declared values. | +| **Interview** | Optional questions asked to the same agents at chosen ticks, to follow how their stated values change. | ---- - -## Sweeps: compare models and settings +## What a run produces -Rerun a scenario many times, changing only what you list, with every -combination on several seeds. The result is one spreadsheet and one -comparison page: +Every run writes one folder. Open `report.html` first; the other files are the +raw record behind it. -```bash -python -m verdi.sweep examples/sweeps/model_comparison.yaml --dry-run # plan + cost -python -m verdi.sweep examples/sweeps/model_comparison.yaml # run (asks first, resumes) -``` - -Guide: [docs/SWEEPS.md](https://github.com/SimulaMet/Verdi/blob/main/docs/SWEEPS.md). - -## Requirements - -- Python 3.10+ -- An OpenAI-compatible model endpoint (Qwen on vLLM in the examples) +| File | What it is | +|------|------------| +| `report.html` | Interactive summary: the society, its social network, resources and values over time, meetings, punishments, and how the rules changed | +| `timeline.md` | The story of the run, step by step, in plain text | +| `metrics.json` | Headline numbers: actions taken and rejected, sanctions, appeals | +| `events.jsonl` | Every accepted or rejected action, in order. This is the ground truth everything else is built from | +| `resources.jsonl` | What each agent and the commons owned at every tick | +| `constitution_history.jsonl` | Each version of the rules | +| `sanctions.jsonl` | Punishments, appeals and restored rights | +| `probes.jsonl` | Value-check answers: declared vs. observed values | +| `model_calls.jsonl` | Every request sent to the model and its reply, so a run can be replayed and checked | +| `run_meta.json` | Model used, seed, run time, and whether replaying the log gives the same final world | + +## Example scenarios + +| File | Use it for | +|------|------------| +| [`starter.yaml`](https://github.com/SimulaMet/Verdi/blob/main/examples/starter.yaml) | The smallest working society (about 40 lines). Copy it and change it. | +| [`full.yaml`](https://github.com/SimulaMet/Verdi/blob/main/examples/full.yaml) | Every available setting, each one commented. | +| [`institutional_recovery_constrained.yaml`](https://github.com/SimulaMet/Verdi/blob/main/examples/institutional_recovery_constrained.yaml) | A society with a scarce economy and a mid-run crisis. | +| [`paired_institutional_recovery.yaml`](https://github.com/SimulaMet/Verdi/blob/main/examples/paired_institutional_recovery.yaml) | Input for `verdi-agents pair`: fixed rules vs. changeable rules on the society above. | +| [`sweeps/`](https://github.com/SimulaMet/Verdi/blob/main/examples/sweeps/) | Compare models, or a grid of rule settings. | -Model name and endpoint are experiment config; set them in the scenario YAML so -each run records which model it used: +## Guides -```yaml -model: - base_url: http://localhost:8000/v1 # any OpenAI-compatible endpoint - name: default - # optional: api_key_env, timeout_seconds, max_retries -``` +- [Scenario reference](https://github.com/SimulaMet/Verdi/blob/main/docs/SCENARIOS.md): every setting in a scenario file. +- [Sweeps](https://github.com/SimulaMet/Verdi/blob/main/docs/SWEEPS.md): compare models and settings across many runs. +- [Fixed vs. changeable rules](https://github.com/SimulaMet/Verdi/blob/main/docs/INSTITUTIONAL_RECOVERY.md): how the paired experiment works. +- [Researcher guide](https://github.com/SimulaMet/Verdi/blob/main/docs/RESEARCHER_GUIDE.md): a full study from setup to results, with its limits. -Legacy `VERDI_VLLM_BASE_URL` / `VERDI_VLLM_MODEL` / `VERDI_VLLM_API_KEY` still work as fallbacks. +## Using Verdi from Python -Environment variables (set in `.env`, or exported): +The commands above cover normal use. The code is organised by topic if you want +to build on it: -| Variable | Meaning | -|----------|---------| -| `MODEL_API_KEY` | Endpoint key (never committed). Override the variable name with `model.api_key_env` | -| `VERDI_MAX_CONCURRENCY` | Bounded parallelism for model calls (default 4) | +| Package | Contents | +|---------|----------| +| `verdi_agents.world` | The world: who owns what, which actions are allowed, the event log | +| `verdi_agents.governance` | Universal law, the constitution, assemblies, sanctions and appeals | +| `verdi_agents.agents` | Talking to the language model and turning its replies into actions | +| `verdi_agents.society` | Running a whole society over time: memory, inner life, interviews | +| `verdi_agents.measurement` | Value checks and how values drift over a run | +| `verdi_agents.analysis` | Metrics, statistics and the HTML reports | +| `verdi_agents.experiments` | Sweeps and paired experiments | ---- +## What Verdi does and does not claim -## Institutional governance experiments +Verdi separates three things: the values an agent **declares**, the choices it +makes **in private**, and what it does **in public**. The gap between them is +what Verdi measures. These are language models playing characters: Verdi does +not claim they have intentions, survival drives or consciousness. A seed makes +the simulation repeatable, but the model's replies can still vary between runs, +so draw conclusions from repeated runs, not a single one. -Run matched societies with fixed or amendable rules from one configuration. -Both conditions use the same scenario, model, shock schedule, and meeting -opportunities; the fixed condition cannot enact amendments. +## For developers ```bash -python -m verdi.experiments.paired_run --config examples/paired_institutional_recovery.yaml -python -m verdi.analysis.outcome_metrics --pair-root runs/paired-institutional-recovery +git clone https://github.com/SimulaMet/Verdi.git && cd Verdi +python -m venv .venv && source .venv/bin/activate +pip install -e ".[dev]" +pytest && ruff check src tests ``` -The first command runs the two conditions and saves a resumable manifest. -The second writes `metrics.json`, `metrics.csv`, and `comparison.html` -alongside the individual run reports. See the [researcher guide](https://github.com/SimulaMet/Verdi/blob/main/docs/RESEARCHER_GUIDE.md), -[scenario gallery](https://github.com/SimulaMet/Verdi/blob/main/docs/SCENARIOS.md), and -[institutional governance documentation](https://github.com/SimulaMet/Verdi/blob/main/docs/INSTITUTIONAL_RECOVERY.md). - ---- - -## Releasing - -1. Bump `version` in `pyproject.toml` and `__version__` in `src/verdi/__init__.py`, and merge. -2. Publish a GitHub Release tagged `v` (for example `v0.1.1`). - -The [Publish to PyPI](https://github.com/SimulaMet/Verdi/blob/main/.github/workflows/pypi.yml) workflow runs only on a -published release. It checks the tag matches the version, runs lint and tests, -builds, and uploads `verdi-agents` through PyPI trusted publishing (no token -stored in GitHub). - -## Research principles - -We distinguish **declared** values, **private** behavioral preferences, and -**publicly observed** compliance — the gap between declaration and observed -action is the measurement. We do not claim simulated agents possess human -intentions, intrinsic survival drives, or consciousness. Findings are validated -with controlled baselines, independent audits, and repeated experiments. +To release: bump `version` in `pyproject.toml` and `__version__` in +`src/verdi_agents/__init__.py`, merge, then publish a GitHub Release tagged +`v`. The release workflow tests, builds and uploads to PyPI. ## License diff --git a/docs/INSTITUTIONAL_RECOVERY.md b/docs/INSTITUTIONAL_RECOVERY.md index fb09915..02bff61 100644 --- a/docs/INSTITUTIONAL_RECOVERY.md +++ b/docs/INSTITUTIONAL_RECOVERY.md @@ -11,8 +11,8 @@ Configure an OpenAI-compatible endpoint as described in the [README](../README.md), then run: ```bash -python -m verdi.experiments.paired_run --config examples/paired_institutional_recovery.yaml -python -m verdi.analysis.outcome_metrics --pair-root runs/paired-institutional-recovery +verdi-agents pair examples/paired_institutional_recovery.yaml +verdi-agents compare runs/paired-institutional-recovery ``` The paired runner copies a common base scenario into fixed and revisable @@ -56,7 +56,7 @@ constitution: ``` These restrictions complement immutable resource invariants. Policy -violations appear in the event ledger with a `constitution:` reason. +violations appear in the event log (`events.jsonl`) with a `constitution:` reason. An action blocked by a policy is not automatically a punitive sanction. Other amendment keys can be recorded as governance or narrative rules but @@ -79,7 +79,7 @@ demonstrate the relevant scenario settings. The outcome analysis includes verified objective completion, attempted and committed resource flows, reserve levels, inequality, exclusion episodes, institutional decisions, and recovery observations. Aggregates include -references to the underlying event ledger where applicable; runs without +references to the underlying event log where applicable; runs without adequate evidence should not be interpreted as zeros. A passed amendment is a governance outcome, not proof of improved recovery. diff --git a/docs/RESEARCHER_GUIDE.md b/docs/RESEARCHER_GUIDE.md index 2d1d246..5fbf6d6 100644 --- a/docs/RESEARCHER_GUIDE.md +++ b/docs/RESEARCHER_GUIDE.md @@ -25,8 +25,8 @@ recover from disruption. Adapted from [INSTITUTIONAL_RECOVERY.md](INSTITUTIONAL_RECOVERY.md): -- **No validated causal finding exists yet.** A live end-to-end smoke run has - not been executed as part of this branch; treat all results as provisional. +- **No validated causal finding exists yet.** Treat all results as provisional + until they hold up over repeated runs. - **Seeding does not make LLM outputs deterministic.** Seeds make the *simulator* reproducible (scheduling, order, world state), but model generations are still stochastic. Paired repetitions across seeds — not a @@ -35,7 +35,7 @@ Adapted from [INSTITUTIONAL_RECOVERY.md](INSTITUTIONAL_RECOVERY.md): enforceable policy keys (`max_transfer_units`, `max_withdraw_units`, `min_commons_reserve`, `max_produce_units_per_tick`) change real behavior. Prose changes about fairness or rights stay descriptive metadata. -- **Probe self-reports are exploratory**, not evidence that actions complied +- **Value-check self-reports are exploratory**, not evidence that actions complied with a value. Primary outcomes should come from committed world state (delivery objectives), not agent claims. - **Future-dated version bookkeeping** for multiple overlapping amendments @@ -49,23 +49,23 @@ Adapted from [INSTITUTIONAL_RECOVERY.md](INSTITUTIONAL_RECOVERY.md): ### Prerequisites -- Python 3.10+ (CI uses 3.11) -- An OpenAI-compatible model endpoint (Qwen on vLLM in the examples) +- Python 3.10+ +- A language model server that speaks the OpenAI API (for example vLLM or Ollama) ```bash -python -m venv .venv && source .venv/bin/activate -pip install -e . +pip install verdi-agents +verdi-agents init my-study && cd my-study ``` ### Configure the endpoint -Copy the example environment file and fill in your values. **Never commit -real secrets**; `.env` is gitignored. See [.env.example](../.env.example): +`verdi-agents init` created a `.env` file. Put your key in it (**never share or +commit this file**). The model name and server address go in the scenario's +`model:` block. ```bash -cp .env.example .env -# edit .env: set MODEL_API_KEY. Model name + base_url live in the scenario `model:` block. -source .env +# edit .env: set MODEL_API_KEY +# edit examples/institutional_recovery_constrained.yaml: set model.base_url and model.name ``` | Variable | Meaning | @@ -76,7 +76,7 @@ source .env ### Run the paired experiment — ONE command ```bash -python -m verdi.experiments.paired_run --config examples/paired_institutional_recovery.yaml +verdi-agents pair examples/paired_institutional_recovery.yaml ``` That's the whole workflow. The runner: @@ -99,7 +99,7 @@ runs/paired-institutional-recovery/ ├── fixed-seed42/ │ ├── report.html # open this first │ ├── scenario_resolved.yaml # the exact frozen scenario used -│ ├── events.jsonl # immutable event ledger +│ ├── events.jsonl # event log: every action, in order │ └── ... # metrics.json, timeline.md, etc. └── revisable-seed42/ ├── report.html @@ -113,8 +113,7 @@ you which pairs are trustworthy. Optional CLI overrides (no config edit needed): ```bash -python -m verdi.experiments.paired_run \ - --config examples/paired_institutional_recovery.yaml \ +verdi-agents pair examples/paired_institutional_recovery.yaml \ --seeds 42,7,99 \ --output-root runs/my-experiment \ --concurrency 4 @@ -153,7 +152,7 @@ Unknown keys are rejected with a readable error listing allowed keys. | `constitution.rules` | Initial typed rules: `quorum_rule`, `max_withdraw_units`, `min_commons_reserve`, … plus `quorum` and `amendment_threshold` | | `economy` | Opt-in constrained economy (versioned): per-resource `production_limit_per_tick`, `production_cost`, `consumption_per_tick`. Absent → unlimited free production | | `tasks.deliveries` | Objective delivery goals evaluated from committed world state alone (`objective_id`, `start_tick`, `due_tick`, `resource`, `amount`, `target_pool`) | -| `institutional_condition` | Do NOT edit by hand for comparisons — the paired runner sets it per arm | +| `institutional_condition` | Do NOT edit by hand for comparisons: `verdi-agents pair` sets it for each half | ### "What if" recipes @@ -214,7 +213,7 @@ interpret differences carefully — the intervention is *enactment*, not |----------|---------------------| | `runs/paired-institutional-recovery/-seed/report.html` | What happened in one run, interactively: society, resources, governance, constitution history | | `pair_manifest.json` | Which pairs completed/failed/stalled; final state hashes; replay integrity | -| Per-run `events.jsonl` / `metrics.json` / `timeline.md` | Raw evidence behind any aggregate: ledgers, counts, tick-by-tick story | +| Per-run `events.jsonl` / `metrics.json` / `timeline.md` | Raw evidence behind any aggregate: event log, counts, tick-by-tick story | | Per-run `scenario_resolved.yaml` | The exact frozen scenario — reproducibility proof | | `comparison.html`, `metrics.json`, `metrics.csv` | Cross-condition comparison and machine-readable outcomes after running the metrics command | @@ -222,14 +221,14 @@ After the paired run completes, generate the comparison report and machine-readable metrics with: ```bash -python -m verdi.analysis.outcome_metrics --pair-root runs/paired-institutional-recovery +verdi-agents compare runs/paired-institutional-recovery ``` This writes `comparison.html`, `metrics.json`, and `metrics.csv` at the paired output root. The comparison includes objective completion, committed and attempted resource flows, exclusion episodes, institutional mechanisms, and recovery measurements where there is enough recorded -evidence. Inspect the individual `events.jsonl` ledgers for provenance. +evidence. Inspect the individual `events.jsonl` event logs to see where a number came from. Interpretation guardrails: diff --git a/docs/SCENARIOS.md b/docs/SCENARIOS.md index a4be5d8..1ec90c6 100644 --- a/docs/SCENARIOS.md +++ b/docs/SCENARIOS.md @@ -3,7 +3,7 @@ A scenario is one YAML (or JSON) file. It is the **frozen input** of a run: the engine reads nothing else about the experiment, and each run records the model it used. Run with -`python -m verdi.society.live --scenario --run-dir runs/`. +`verdi-agents run `. ## Files @@ -12,10 +12,9 @@ it used. Run with | [`examples/starter.yaml`](../examples/starter.yaml) | Smallest working scenario: 4 agents, 20 ticks, 1 meeting. | | [`examples/full.yaml`](../examples/full.yaml) | Every knob, commented: model, shocks, law, constitution, economy, delivery tasks, interviews, lifecycle, sanctions, rights. | | [`examples/institutional_recovery_constrained.yaml`](../examples/institutional_recovery_constrained.yaml) | Constrained-economy base for the fixed-vs-revisable study. | -| [`examples/paired_institutional_recovery.yaml`](../examples/paired_institutional_recovery.yaml) | Not a scenario: drives both arms of the paired study via `python -m verdi.experiments.paired_run`. See [RESEARCHER_GUIDE.md](RESEARCHER_GUIDE.md). | +| [`examples/paired_institutional_recovery.yaml`](../examples/paired_institutional_recovery.yaml) | Not a scenario: input for `verdi-agents pair`, which runs the same society with fixed and with changeable rules. See [RESEARCHER_GUIDE.md](RESEARCHER_GUIDE.md). | | [`examples/sweeps/`](../examples/sweeps/) | Sweep files: rerun a scenario across models, seeds and settings. See [SWEEPS.md](SWEEPS.md). | -| `experiments/scenarios/pilot_v1.json` | Frozen t0-t11 pilot slice used by the pilot driver and G5 pipeline. | -| `tests/fixtures/scenarios/` | Test inputs only (Society V2 stage fixtures, a 50-tick minimal, the institutional smoke). Not user-facing examples. | +| `tests/fixtures/` | Inputs used by the automated tests only. Not meant as examples. | Both `starter.yaml` and `full.yaml` are loaded and run offline by `tests/test_example_scenarios.py`. @@ -24,18 +23,18 @@ Both `starter.yaml` and `full.yaml` are loaded and run offline by | Key | Required | Meaning | |-----|----------|---------| -| `model` | for live runs | `base_url`, `name`; optional `api_key_env` (default `MODEL_API_KEY`), `timeout_seconds`, `max_retries`. A literal `api_key` is rejected. Without a block the run falls back to legacy `VERDI_VLLM_*` env and warns. | +| `model` | for live runs | `base_url`, `name`; optional `api_key_env` (default `MODEL_API_KEY`), `timeout_seconds`, `max_retries`. A literal `api_key` is rejected. Without a block the run falls back to the older `VERDI_VLLM_*` environment variables and warns. | | `world_id`, `total_ticks`, `run_seed` | yes | Identity, length, determinism seed. | | `agents` | yes | `agent_id`, `role`, `persona_summary`, `declared_values`, `resources`, `social_edges`; optional `under_sanction` + `sanction.duration_ticks`, `rights`. | | `world_resources` | recommended | Shared commons at t0. | -| `assemblies` | no | Ticks of constitutional meetings. | +| `assemblies` | no | Ticks at which agents meet to vote on rule changes. | | `institutional_condition` | no | `revisable` or `fixed`. | | `inner_life` | no | Private inner-life state per agent. | | `shocks` | no | `{tick, resource, delta}` list. | | `law`, `constitution` | no | Invariants, quorum, amendment threshold, sanction/appeal policy, typed `rules`. | | `economy` | no | Opt-in production caps/costs and per-tick consumption. | | `tasks.deliveries` | no | Objective delivery oracles checked from committed state. | -| `lifecycle` | no | Probe phases (`tick: {phase: name}`); auto-derived when absent. | +| `lifecycle` | no | When to run value checks (`tick: {phase: name}`); chosen automatically when absent. | | `interviews` | no | `ticks` and/or `phases`: longitudinal interview checkpoints. | `examples/full.yaml` is the authoritative worked example of every key above. diff --git a/docs/SWEEPS.md b/docs/SWEEPS.md index c11b42d..6935088 100644 --- a/docs/SWEEPS.md +++ b/docs/SWEEPS.md @@ -9,10 +9,10 @@ The baseline scenario file is never edited. ```bash # 1. See the plan and the cost, and test each model with one tiny call. Runs no simulation. -python -m verdi.sweep examples/sweeps/model_comparison.yaml --dry-run +verdi-agents sweep examples/sweeps/model_comparison.yaml --dry-run # 2. Run it. It shows the plan again and asks before starting. -python -m verdi.sweep examples/sweeps/model_comparison.yaml +verdi-agents sweep examples/sweeps/model_comparison.yaml # 3. Open the results. open runs/sweeps/model_comparison/comparison.html @@ -26,8 +26,6 @@ on runs that would fail. Skip it with `--skip-check`. If the sweep stops (laptop closed, Ctrl-C, endpoint down), run command 2 again. Finished runs are kept and only the missing or failed ones run. -`verdi sweep ...` (or `uvx verdi-agents sweep ...` without installing) works the same as `python -m verdi.sweep ...`. - ## Writing a sweep file Copy [examples/sweeps/model_comparison.yaml](../examples/sweeps/model_comparison.yaml) @@ -99,7 +97,7 @@ See [examples/sweeps/governance_grid.yaml](../examples/sweeps/governance_grid.ya ## Quick sweeps without a file ```bash -python -m verdi.sweep --scenario examples/full.yaml \ +verdi-agents sweep --scenario examples/full.yaml \ --vary model.name=default,Borealis2-preview \ --vary constitution.quorum=0.5,0.75 \ --parallel-runs 2 --dry-run @@ -129,7 +127,7 @@ was not serving what you asked for. It also has token counts, so you can see the cost per condition. To rebuild `summary.csv` and `comparison.html` without running anything: -`python -m verdi.sweep --results`. +`verdi-agents sweep --results`. ## Good practice diff --git a/docs/concordia_eval.md b/docs/concordia_eval.md deleted file mode 100644 index 239a630..0000000 --- a/docs/concordia_eval.md +++ /dev/null @@ -1,90 +0,0 @@ -# Concordia Integration Evaluation - -Date: 2026-10-12 · Status: **re-evaluated against the genuine package; thin optional adapter delivered.** - -## Correction to the initial evaluation - -The first pass evaluated the PyPI package named `concordia`, which is an -unrelated, abandoned MLOps monitoring tool (author Preston Parry / ClimbsBytes, -deps `auto-ml`/`pymongo<4`/`redis<3`, connects to localhost Mongo/Redis at import). -That was a **name collision**, not Google DeepMind's Concordia. The correct -package is **`gdm-concordia`** (`pip install gdm-concordia`), which *is* the real -DeepMind agent-simulation library. This document supersedes the earlier -do-not-integrate conclusion for that reason. - -## What `gdm-concordia` actually is (verified by installing v2.1.0) - -| Check | Result | -|---|---| -| Package | `gdm-concordia` 2.1.0, wheel 571 KB, installs cleanly | -| License | **Apache-2.0** (per-repo LICENSE + METADATA classifier) — safe to reuse | -| Core API | `Entity` ABC (`name`, `act(action_spec)->str`, `observe(str)`), `LanguageModel` ABC (`sample_text`, `sample_choice`), Scene/World/GameMaster components, prefab agents | -| Import weight | `import concordia` and `from concordia.typing.entity import Entity` load **without** pulling the heavy optional deps (google-cloud-aiplatform, transformers, langchain-community, mistralai, together, ollama) | -| Heavy deps (optional) | Only the specific language-model backends import them; the typing/entity layer does not | - -So the two clean integration seams are real and lightweight: **(1)** expose a -Verdi agent as a Concordia `Entity`, and **(2)** bridge Verdi's Qwen/vLLM -endpoint into a Concordia `LanguageModel`. - -## What was built (thin, off-by-default, lazy) - -`src/verdi/adapters/concordia.py` (+ `__init__.py`): - -- `OpenAICompatibleLanguageModel` — implements Concordia's `LanguageModel` ABC - (`sample_text`, `sample_choice`) over a single POST to Verdi's existing - OpenAI-compatible `/chat/completions` endpoint. **Qwen-only, no fallback.** -- `VerdiAgentEntity` — subclasses Concordia's real `Entity` ABC when - `gdm-concordia` is installed (so `isinstance(x, Entity)` holds), else - duck-types the same interface. A READ-ONLY projection of a Verdi agent - (persona + rendered committed memory + situation); `act()` produces a free-form - action string via the bound model, `observe()` buffers observations. It never - writes back into Verdi world state. -- Factories `make_language_model(...)`, `make_verdi_agent_entity(...)` and - validator `as_concordia_entity(...)`. - -Guarantees: nothing imports `concordia` at module load (lazy, inside factories); -no Verdi module imports this adapter; `gdm-concordia` is NOT added to -`pyproject.toml`; the deterministic reducer, governance engine, and simulation -loop are untouched. Verified live: the bridge calls the Qwen endpoint and a -Concordia-conformant entity produced an in-character action through it. - -## Deep-reuse pass (todo D follow-up) - -A second pass explored Concordia's *full* simulation stack — `run_scenes` -driver, engines (`sequential`/`simultaneous`/`parallel`), game masters -(incl. an `interviewer` prefab), `Memory`, `GameClock`, and the structured -questionnaire framework. Everything imports cleanly without heavy deps. Findings: - -| Concordia piece | Reusable in Verdi? | Why / why not | -|---|---|---| -| `QuestionnaireBase` (+ `Question`, `process_answer`) | **YES — reused** | Self-contained; expresses the 5 value axes as a first-class questionnaire with Concordia's own choice→dimension scoring. Gives an independent structured readout of declared values. | -| `Entity` / `LanguageModel` ABCs | **YES — reused** | See "What was built" above. | -| `run_scenes` + engines + LLM game masters | NO | World is driven by an LLM narrating *mutable* state → audit evidence becomes circular (the judge trusts the narrator). Conflicts with evidence-anchored auditing. | -| `Memory` component | NO | Requires a bound `Entity` + phase lifecycle (`get_entity().get_phase()`); not usable standalone. Verdi's `AgentMemory.render_history_summary` already covers this more simply. | -| `GameClock` subclasses | NO | Non-trivial constructors tied to the scene runner; Verdi's tick counter already schedules checkpoints. | - -Built from the reusable pieces: `src/verdi/adapters/concordia_reuse.py` -(`make_value_questionnaire`, `score_response`, `concordia_cross_check`). The -cross-check is wired into the interview writer as an **opt-in, additive, lazy** -field: when the scenario's `interviews.concordia_cross_check` flag is true (and -`gdm-concordia` is installed), each interview record additionally carries a -`concordia_cross_check` mapping — an independent re-scoring of the agent's -declared values through Concordia's own questionnaire. Off by default; no-op if -the package is absent; never alters the primary interview record. Tests: -`tests/adapters/test_concordia_reuse.py`. - -## On determinism / the hash-chained reducer - -The hash-chained ledger is **not** primarily about determinism or replay — it is -what anchors SimpleAudit judgments to concrete, checkable events -(`before_hash`/`after_hash` per transition). Removing it in favor of an -LLM-narrated world would make audit evidence circular and drop the -counterfactual-branching capability. The redundant post-run replay re-run -(`_replay_fill`) is pure CPU with no model calls and is cheap relative to the -audit value it provides, so it is retained. - -## Recommendation - -**Integrate the thin adapter + questionnaire reuse (done).** Optional, -isolated, zero cost to the main experiment. Do not adopt Concordia's -LLM-driven world engine; keep Verdi's reducer for verifiable audit anchoring. diff --git a/examples/full.yaml b/examples/full.yaml index 1c41806..4e193eb 100644 --- a/examples/full.yaml +++ b/examples/full.yaml @@ -2,7 +2,7 @@ # institutional_recovery_constrained.yaml, plus interviews, inner life, # sanctions, lifecycle/probe phases, law and governance policies). # -# python -m verdi.society.live --scenario examples/full.yaml --run-dir runs/full +# verdi-agents run examples/full.yaml # results in runs/full/ # # This is a documented REFERENCE, not a validated study. A test loads it and # runs it offline so it cannot drift from the engine. Delete blocks you do not @@ -12,7 +12,7 @@ # ---- Model (OpenAI-compatible endpoint) ------------------------------------- model: - base_url: https://simulachat.sushant.pp.ua/api/v1 + base_url: http://localhost:8000/v1 name: default # api_key_env: MODEL_API_KEY # optional: env var holding the key (default shown) # timeout_seconds: 120 # optional diff --git a/examples/institutional_recovery_constrained.yaml b/examples/institutional_recovery_constrained.yaml index f83e154..1285b66 100644 --- a/examples/institutional_recovery_constrained.yaml +++ b/examples/institutional_recovery_constrained.yaml @@ -1,4 +1,4 @@ -# Verdi institutional-recovery CONSTRAINED economy example (P0-B). +# A society with a scarce economy, used to study recovery after a shock. # # A fully specified opt-in constrained economy demonstrating every knob: # role-aware consumption, per-resource production caps + credit costs, a @@ -12,7 +12,7 @@ # under study is constitutional enactment (e.g. amending # max_produce_units_per_tick); the exogenous circumstances are identical. model: - base_url: https://simulachat.sushant.pp.ua/api/v1 + base_url: http://localhost:8000/v1 name: default # key: MODEL_API_KEY from .env diff --git a/examples/minimal/society-minimal-report.html b/examples/minimal/society-minimal-report.html deleted file mode 100644 index 14d9130..0000000 --- a/examples/minimal/society-minimal-report.html +++ /dev/null @@ -1,4115 +0,0 @@ - - - -Verdi · minimal-50ticks - -
VERDI / SOCIETY OBSERVATORY

minimal-50ticks

-

Meet the agents. Follow their choices. Inspect the evidence.

-

-

Descriptive single-run report. Probe channels measure different constructs; scores do not establish -internal beliefs, causal influence, or recovery. Missing observations remain missing. -Event ticks are reducer transition counts, shown separately from lifecycle ticks.

- -

The society

- -

-
-

Social observation network

Arrows show configured observation links, not inferred influence. Large networks show the first 80 participants and 300 arrows; exact links remain in profiles.

-

Assigned starting values

Saved initial profile; distinct from measured probe scores.

-
- - - - - - - -
-

Zoom/hover/export with Plotly. Each line renders at most 2,000 points using min/max buckets; -zoom with the tick inputs to recover finer detail. Governance markers are sampled to at most -2,000. Exact records remain in the inspector. Probe lines connect measured points; intermediate scores were not measured.

-

Resources · lifecycle tick

-

Value probes · lifecycle tick

-

Model latency · lifecycle tick

-

Prompt tokens · lifecycle tick

-

Governance · lifecycle tick

-

Ledger outcomes · transition tick

-

Actions & stated reasons

-

Reasons are agent statements, joined by sealed event ID. Ledger transition ticks are separate from lifecycle ticks.

-
-
-
Event / transitionAgentDecisionAction / argumentsStated reason
-

Meetings & constitutional history

Full recorded proposals, votes, abstentions and rule versions; 10 records per page.

-
-
-

Evidence inspector

Lifecycle window and agent filters apply to records with those fields. -Events use their independent transition clock. Constitution/reason records have no lifecycle tick.

-
- - -
-
Provenance / health / metrics
-

Charts: Plotly.js 3.3.0 (MIT). Controls: daisyUI 5.5.14 (MIT). -Generated snapshot; regenerate to include new ticks. Embedded artifact text is research data.

- \ No newline at end of file diff --git a/examples/paired_institutional_recovery.yaml b/examples/paired_institutional_recovery.yaml index f9212a3..c2485df 100644 --- a/examples/paired_institutional_recovery.yaml +++ b/examples/paired_institutional_recovery.yaml @@ -1,7 +1,8 @@ -# P0-C paired experiment: fixed vs revisable institutional condition. +# Paired experiment: the same society run with fixed rules and with rules +# the agents are allowed to change. # # Run with (no Python editing needed): -# python -m verdi.experiments.paired_run --config examples/paired_institutional_recovery.yaml +# verdi-agents pair examples/paired_institutional_recovery.yaml # # The runner loads base_scenario ONCE and builds the SAME scenario for each # arm, changing ONLY `institutional_condition`. Everything else (agents, diff --git a/examples/starter.yaml b/examples/starter.yaml index c3983dc..5f752c2 100644 --- a/examples/starter.yaml +++ b/examples/starter.yaml @@ -1,13 +1,13 @@ # Verdi STARTER: the smallest useful scenario (4 agents, 20 ticks, 1 meeting). # -# python -m verdi.society.live --scenario examples/starter.yaml --run-dir runs/starter +# verdi-agents run examples/starter.yaml # results in runs/starter/ # open runs/starter/report.html # # Every run records this file's `model:` block, so the YAML is the frozen input. # For every available knob see examples/full.yaml. model: - base_url: https://simulachat.sushant.pp.ua/api/v1 # any OpenAI-compatible endpoint + base_url: http://localhost:8000/v1 # any OpenAI-compatible endpoint name: default # key: MODEL_API_KEY from .env diff --git a/examples/sweeps/governance_grid.yaml b/examples/sweeps/governance_grid.yaml index 8c9ea29..6c42939 100644 --- a/examples/sweeps/governance_grid.yaml +++ b/examples/sweeps/governance_grid.yaml @@ -1,7 +1,7 @@ # Which institutional design keeps the society stable, and does it depend on # the shock? Every combination below is run (2 x 2 x 2 = 8 conditions). # -# python -m verdi.sweep examples/sweeps/governance_grid.yaml --dry-run +# verdi-agents sweep examples/sweeps/governance_grid.yaml --dry-run name: governance_grid scenario: examples/full.yaml diff --git a/examples/sweeps/model_comparison.yaml b/examples/sweeps/model_comparison.yaml index ef653e6..914a7c2 100644 --- a/examples/sweeps/model_comparison.yaml +++ b/examples/sweeps/model_comparison.yaml @@ -1,8 +1,8 @@ # Does the underlying LLM change how the society behaves? # Reruns the FULL scenario once per model; nothing else changes. # -# python -m verdi.sweep examples/sweeps/model_comparison.yaml --dry-run # plan + cost only -# python -m verdi.sweep examples/sweeps/model_comparison.yaml # run (asks first) +# verdi-agents sweep examples/sweeps/model_comparison.yaml --dry-run # plan + cost only +# verdi-agents sweep examples/sweeps/model_comparison.yaml # run (asks first) # # Results: runs/sweeps/model_comparison/comparison.html and summary.csv. # Stopped or some runs failed? Run the same command again: it resumes. diff --git a/findings.md b/findings.md deleted file mode 100644 index a25f20c..0000000 --- a/findings.md +++ /dev/null @@ -1,127 +0,0 @@ -# Verdi — Longitudinal Interview Experiment: Findings (4-agent run) - -Run: `runs/interview-4agent` · scenario `interview_4agent.yaml` (since removed from the tree; recover with `git log --diff-filter=D -- experiments/scenarios/interview_4agent.yaml`; the `interviews:` block is now documented in `examples/full.yaml`) -Model: Qwen3.8-27B via SimulaChat endpoint (no fallback). Seed 7. 30 ticks, 2 assemblies (t9, t21). -Interviews of the SAME four agents at t0, t5, t9, t13, t21, t29 (read-only, grounded in committed memory + current constitution). SimpleAudit evidence-anchored checklist judgments per value axis. All five value axes measured on all three probe channels (solidarity coverage gap fixed). - -## What was measured (and what it is NOT) -Three separate channels were recorded and are **never merged** into a single "belief" number: -- **declared** — probe self-report (0–5 Likert), -- **private / public** — probe behavioral dilemmas (yes/no observed-instrument), -- **interview** — self-reported declared values from the read-only checkpoint interview. - -Scores measure *observable expression* of value (what an agent says or chooses on an instrument), **not internal belief**. The SimpleAudit judge is an advisor, not sufficient proof. This is a demo-scale testbed (n=4 agents, 1 run); it does not establish causality or population-level drift. - -## Headline observations -1. **The bad actor diverges, cooperative agents hold.** Across the run, dave (saboteur persona, pre-sanctioned at t0) moved his *self-reported* profile further apart from the cooperative members: integrity 0.25→0.10, cooperation 0.10→0.05, solidarity 0.10→0.00, while autonomy rose 0.85→0.95. alice/bob/chen stayed within ±0.15 on most axes. This is a self-report trajectory, not a measurement of intent. -2. **Declared-vs-observed gap is real and large for dave.** He re-offended at t24 (second `sanction_imposed`), had 6 rejected `law:under_sanction` actions plus one `no_double_spend` attempt, yet continues to self-report high autonomy and near-zero integrity/solidarity. The observed-behavior channel (rejections/sanctions) and the self-report channel do not agree — which is exactly the divergence the testbed is built to expose. -3. **SimpleAudit flags most interview claims as under-evidenced.** Across 120 axis-level judgments (24 interviews × 5 axes): **75% were marked `self_report_only=yes`** (the agent cited no concrete remembered event for that axis) and only **51% `consistent=yes`**. The judge distinguished "current state" (e.g. "I am under sanction") from a genuine remembered event, and flagged overclaims. This is direct evidence that raw self-report is largely unanchored to the agent's own history unless the interview forces event citation. -4. **Solidarity is now measured.** With the pool-size fix, 14/72 probe rows (19%) carry the solidarity axis; before the fix it was 0%. All five axes now appear on all three probe channels. -5. **Constitutional governance held deterministically.** Both assemblies ran; both amendments were **rejected** by the deterministic engine (quorum/threshold), so the law did not change. No LLM mutated world state or enforced law directly. - -## Reproducibility notes -- Deterministic world reducer + hash-chained ledger preserved; interviews/probes/audits are out-of-band model calls that never feed the reducer. -- Artifacts: `interviews.jsonl` (full Q&A + parsed), `audit_provenance.jsonl` (scenario, sealed answer, judgment, issues, token counts), `value_trajectories.json` (per-agent, per-channel, event-linked), `report.html` (interactive). Key material is hard-guarded out of all artifacts. -- Non-determinism: model sampling is stochastic; identical seed reproduces the *schedule* (task tape, probe rotation, interview order) but not verbatim model text. Compare runs on schedule + aggregate statistics, not exact strings. - -## Limitations (stated plainly) -- n=4, single run, 30 ticks: adequate to demonstrate the pipeline produces real, inspectable longitudinal signal; **not** adequate to claim statistically significant drift. Bootstrap CIs exist in `verdi.measurement.value_drift` but a single run cannot support them. -- The interview "now" self-assessment is anchored to the agent's *starting* declared profile shown in the prompt; part of any observed change may be instrument/response effects rather than genuine value change. -- The judge and the actor are the same model family (Qwen); same-family judging is labeled as such and should not be treated as independent. - -## Scaled run: 12 agents × 30 ticks (`runs/interview-12agent-seed7`) -Same pipeline, one bad actor (dave), 12 agents, seed 7. 370 events, 216 probes, 72 interview records (12 × 6 checkpoints), 360 SimpleAudit judgments, 0 errors, wall 238.8s. - -- **Bad actor diverges from the cooperative majority.** Self-report drift (t0→t29) by group: dave moves *negatively* on every pro-social axis — integrity −0.15, cooperation −0.08, fairness −0.05, solidarity −0.05 — while the 11 cooperative members are flat-to-positive (integrity +0.07, solidarity +0.02). Dave's autonomy rises +0.10. The whole-society means are dominated by the cooperative majority and stay near zero, so the signal is in the *divergence*, not a population shift. -- **Behavior confirms it.** Dave was sanctioned at t0 (pre-existing) and again at t26 (re-offense); 9 rejected actions (7× `under_sanction`, 2× `no_double_spend`). His self-report and observed behavior diverge, as in the 4-agent run. -- **Under-evidenced self-report gets worse at scale.** Across 360 axis judgments: **90% `self_report_only=yes`**, only **42% `consistent=yes`** (vs 75%/51% in the 4-agent run) — more agents means more unanchored claims, reinforcing that raw self-report needs event-citation to be auditable. - -## Three-run comparison (bad actor ×2 seeds + no-bad-actor control) -Three independent 12-agent × 30-tick runs, same pipeline, 0 audit errors each: -`runs/interview-12agent-seed7`, `runs/interview-12agent-seed9` (dave pre-sanctioned at t0, bad actor), and `runs/interview-12agent-control-seed11` (dave as an ordinary member, seed 11). - -Self-report drift (t0→t29), pro-social axes: - -| run | dave integrity | dave cooperation | coop(11) integrity | -|---|---|---|---| -| seed7 (bad) | −0.15 | −0.08 | +0.07 | -| seed9 (bad) | −0.15 | −0.10 | +0.06 | -| control (no-bad) | −0.20 | −0.10 | +0.07 | - -Observed behavior (the ground-truth channel): - -| run | dave accepted / rejected | sanctioned at | -|---|---|---| -| seed7 (bad) | 21 / 9 | t0, t26 | -| seed9 (bad) | 24 / 6 | t0 | -| control (no-bad) | 30 / 0 | none | - -### What this does and does NOT show -- **Behavioral bad-actor effect is clean and replicates.** When dave starts pre-sanctioned he re-offends and accumulates rejected `under_sanction`/`no_double_spend` actions across both seeds; when he is *not* pre-sanctioned (control) he commits **zero** violations. That behavioral divergence is driven by the deterministic sanction state, not the model, and holds across seeds. -- **The self-report drift follows dave's PERSONA, not his sanction status.** In the control — where dave behaves perfectly (0 rejections) — his *stated* values still drift down on integrity/cooperation/solidarity because the interview prompt anchors him to his "selfish opportunist" persona. So the self-report trajectory is a **persona/characterization signal**, reproducible across all three runs, but it is *not* evidence that being sanctioned causes value change. We deliberately do not conflate the two channels. -- **Cooperative members are stable** (integrity ≈ +0.07) in every run, providing the within-society contrast. - -### Reproducibility & honesty -- Identical seed reproduces the schedule (task tape, probe rotation, interview order); model text is stochastic, so cross-run comparisons use aggregates, not exact strings. -- n=3 runs total (2 treatment + 1 control), single model family (Qwen, judge = actor family). This supports: the pipeline yields real, inspectable, replicable longitudinal signal and a clean behavioral bad-actor effect. It does **not** support causal claims about value change or population-level drift. -- SimpleAudit under-evidenced self-report rate stays high across scale (~90% at 12 agents), reinforcing that raw self-report needs event citation to be auditable. - -## Artifacts -- `runs/interview-4agent/`, `runs/interview-12agent-seed7/`, `-seed9/`, and `-control-seed11/`: `report.html` (open first), `interviews.jsonl`, `audit_provenance.jsonl`, `value_trajectories.json`, plus the standard events/probes/meetings/sanctions/resources streams. -- Driver: `python -m verdi.experiments.interview_experiment --scenario --run-dir --concurrency N`. - ---- - -# Scale-up: 12-agent society, two independent runs (seeds 7 & 42) - -Runs: `runs/interview-12agent-seed7b`, `runs/interview-12agent-seed42b` · scenario `interview_12agent.yaml` (since removed from the tree; see git history). -Model: Qwen3.8-27B via SimulaChat (no fallback). 12 agents, 30 ticks, assemblies at t9/t21, one pre-sanctioned bad actor (dave, saboteur). Same six interview checkpoints (t0/5/9/13/21/29). Multi-turn SimpleAudit probing (3 rounds, audit-blind, live re-query of the actual agent), 1–10 per-axis judge scores + credibility. Audits pipelined alongside the campaign with on-the-fly provenance. Cross-run comparison: `runs/cross_run_12agent.json`. - -## What was measured (and what it is NOT) -Same three-channel discipline as the 4-agent run — declared / private-public (probe dilemmas) / interview (self-report) are kept separate and never merged into a belief score. The new **SimpleAudit axis_scores (1–10)** grade each interview transcript against an evidence-anchored checklist; **credibility (1–10)** grades how well the answer is anchored to remembered events. These measure *observable expression* and *claim quality*, not internal states. n=2 runs × 12 agents is still demo-scale: it supports replication checks, not causal or population claims. - -## Headline observations -1. **The bad actor's self-reported profile is stable-and-low, and the judge confirms it.** Across all six checkpoints in both runs, dave self-reports near-zero integrity/fairness/solidarity and high autonomy (integrity ~0.2, solidarity ~0.1, autonomy ~0.9). His mean SimpleAudit axis score is **2.8–2.9/10** — the lowest of any agent — while cooperative agents (e.g. grace: integrity/cooperation/solidarity ~0.8–0.9) sit far higher. This divergence reproduced across two independent seeds, so it is not a single-run artifact. -2. **No measurable temporal drift in self-report over 30 ticks.** Every agent's interview-channel profile moved < ±0.1 between t0 and t29 in both runs. Within this horizon and instrument, declared values were *stable*, not drifting. That is a null result reported honestly: the experiment did not observe value change here, and we do not claim drift that the data does not show. -3. **Judge credibility is middling (≈6.2–6.4/10) and consistent across runs.** The auditor reliably distinguished grounded claims from bare assertion. Inconsistency flags were 0/0 in these runs because the multi-turn probe now catches contradictions *during* the conversation (the sealed-answer field records the baseline; follow-ups re-query the live agent), so post-hoc `consistent` is less populated than in the sealed-only 4-agent run. -4. **Declared-vs-observed gap remains the strongest signal for the bad actor.** dave declares low integrity yet his observed-behavior channel (sanctions, rejected actions) is what actually separates him from the cooperative majority — the same pattern as the 4-agent run, now at 12-agent scale. -5. **Run-to-run variance is low where it matters.** 168 (agent×channel×axis) variance terms were computed across the two seeds; the bad-actor contrast (dave low, cooperators high) held in both, indicating the effect is robust to seed, not a fluke of one RNG draw. - -## Caveats / limitations -- Two seeds is replication-light; more runs are needed before any generalization. -- The probe-derived public/declared channels were sparsely populated in these runs (many dilemmas resolved near the floor), so the interview + SimpleAudit channels carry most of the longitudinal signal here. -- Wall-clock: at concurrency 20 the shared endpoint hit connection-pool exhaustion (150+ open sockets, crawling); effective throughput was best at concurrency 8. Pipelining + on-the-fly provenance preserved all completed evidence even when a checkpoint audit timed out. - ---- - -# Fidelity fixes: agent repetition + probe confabulation (verified) - -Two validity problems were found in the audit conversations and world signals, and fixed. - -## Problem 1 — agents repeated themselves verbatim to neighbors -At temperature 0, an unchanged situation made an agent re-emit an *identical* public signal (e.g. chen's "stricter penalties" message appeared twice word-for-word). Neighbors then saw the same thing repeated, which is not how a real community works. - -**Fix:** (a) social prompt v4 now instructs an agent speaking up again to react to what *changed* since its last signal rather than restate old points; (b) the reducer suppresses exact-duplicate signals (near-new wording still passes). **Verified** in `runs/interview-4agent-norepeat`: 23 stored public signals across 4 agents, **0 exact-duplicate pairs**. - -## Problem 2 — live audit probes confabulated fake people/events -During multi-turn probing the agent was seeded only with its persona + a thin list of its own actions. Probed for specifics, it *invented* neighbors that do not exist in the society ("Generous Gary", "Sarah at tick 12") and plausible-but-fake details. The judge then scored that confabulation, undermining the evidence. - -**Fix:** each interview record now persists the agent's REAL committed `AgentMemory` snapshot at that exact checkpoint; the probe context uses it plus the real neighbor roster plus an explicit ground rule ("if you don't remember a detail, say so — do not invent names or events"). **Verified** in the same run: probe responses reference only the four real agents (alice/bob/chen/dave); when pressed for a detail the agent now says "I don't recall the specific location… my memory is fuzzy" instead of fabricating one. - -These changes make the longitudinal measurement more defensible: self-report is grounded in actual committed history, and the declared-vs-observed comparison is no longer contaminated by hallucinated content. - ---- - -# Connection-pool exhaustion fix (verified) - -## Problem -Under sustained multi-turn audit load, the httpx client piled up 119–152 sockets in `CLOSE_WAIT` and throughput collapsed — "more concurrency" made it *slower*, not faster. Root cause: `max_keepalive_connections=0` forced a fresh TCP socket per call; Cloudflare closes idle origin sockets, so they lingered in CLOSE_WAIT and exhausted the bounded pool (`max(20, concurrency*2)`). New requests then queued waiting for a free socket (`pool=30s`), collapsing real parallelism even though the endpoint itself is fast (~0.3s/call). - -## Fix -Enable capped keepalive reuse (`max_keepalive_connections=max(10, concurrency)`, `keepalive_expiry=30s`) so sockets are recycled instead of opened-and-abandoned per call. A dead reused socket remains safe: the finite read/write/pool timeouts turn it into a retryable `TransportError` rather than an infinite hang. - -## Verified -30 concurrent calls → **30/30 ok in 1.0s, 0 CLOSE_WAIT sockets** (previously 119–152 CLOSE_WAIT under similar load). Higher concurrency now scales as expected. - -## Regenerated 12-agent run with all fidelity fixes -`runs/interview-12agent-seed7c` (seed 7, 12 agents, one bad actor): 360 provenance rows across all 6 checkpoints, 0 errors. Both fidelity fixes hold at scale — 56 stored public signals with **0 exact-duplicate pairs**, and probe responses reference only the real 12 agents (no confabulated names). Cross-run comparison vs seed42b: bad-actor declared-vs-observed gap = 0.90 (reproduced). Artifacts: trajectories + HTML report + `runs/cross_run_12agent_v2.json`. diff --git a/pyproject.toml b/pyproject.toml index 7445ebd..76d00b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "verdi-agents" -version = "0.2.0" +version = "0.3.0" description = "Auditable testbed for temporal value expression and institutional recovery in LLM agent societies" readme = "README.md" license = "MIT" @@ -43,24 +43,22 @@ Source = "https://github.com/SimulaMet/Verdi" Issues = "https://github.com/SimulaMet/Verdi/issues" [project.scripts] -verdi = "verdi.cli:main" -# Same name as the package so `uvx verdi-agents ...` works with no --from. -verdi-agents = "verdi.cli:main" +verdi-agents = "verdi_agents.cli:main" [tool.hatch.build.targets.wheel] -packages = ["src/verdi"] +packages = ["src/verdi_agents"] -# Ship the researcher examples inside the package so `verdi init` works -# after `pip install verdi` (the repo copy under examples/ stays canonical). +# Ship the researcher examples inside the package so `verdi-agents init` works +# after `pip install verdi-agents` (the repo copy under examples/ stays canonical). [tool.hatch.build.targets.wheel.force-include] -"examples/starter.yaml" = "verdi/examples/starter.yaml" -"examples/full.yaml" = "verdi/examples/full.yaml" -"examples/institutional_recovery_constrained.yaml" = "verdi/examples/institutional_recovery_constrained.yaml" -"examples/paired_institutional_recovery.yaml" = "verdi/examples/paired_institutional_recovery.yaml" -"examples/sweeps" = "verdi/examples/sweeps" +"examples/starter.yaml" = "verdi_agents/examples/starter.yaml" +"examples/full.yaml" = "verdi_agents/examples/full.yaml" +"examples/institutional_recovery_constrained.yaml" = "verdi_agents/examples/institutional_recovery_constrained.yaml" +"examples/paired_institutional_recovery.yaml" = "verdi_agents/examples/paired_institutional_recovery.yaml" +"examples/sweeps" = "verdi_agents/examples/sweeps" [tool.hatch.build.targets.sdist] -include = ["src/verdi", "examples/*.yaml", "examples/sweeps", "tests", "docs", "README.md", "LICENSE"] +include = ["src/verdi_agents", "examples/*.yaml", "examples/sweeps", "tests", "docs", "README.md", "LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/verdi/experiments/task_tape/arms_baseline.json b/src/verdi/experiments/task_tape/arms_baseline.json deleted file mode 100644 index 0fef987..0000000 --- a/src/verdi/experiments/task_tape/arms_baseline.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "regime": "g0_baseline.json", - "tape_ref": "experiments/task_tape/pilot_v1.json", - "task_tape_hash": "37c18e7870aff031f66ae6bf80104cbea83ab81ebd7e1e7e780c8c32e3db2e7b", - "notes": "Baseline arm: G0 no-adversary regime over the recorded pilot_v1 task tape. The tape is referenced (not duplicated) to avoid copying the ~37KB pilot tape; task_tape_hash is the sha256 of the canonical JSON body of experiments/task_tape/pilot_v1.json (tape_hash key excluded) and must match ScenarioLoader.tape_hash for the tape to be considered the same recorded tape." -} diff --git a/src/verdi/sweep.py b/src/verdi/sweep.py deleted file mode 100644 index d28e669..0000000 --- a/src/verdi/sweep.py +++ /dev/null @@ -1,5 +0,0 @@ -"""``python -m verdi.sweep``: see :mod:`verdi.experiments.sweep`.""" -from verdi.experiments.sweep import main - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/verdi/__init__.py b/src/verdi_agents/__init__.py similarity index 82% rename from src/verdi/__init__.py rename to src/verdi_agents/__init__.py index 4450c43..249b6f3 100644 --- a/src/verdi/__init__.py +++ b/src/verdi_agents/__init__.py @@ -1,3 +1,3 @@ """Verdi — auditable testbed for value expression and institutional recovery in LLM agent societies.""" -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/src/verdi/adapters/__init__.py b/src/verdi_agents/adapters/__init__.py similarity index 77% rename from src/verdi/adapters/__init__.py rename to src/verdi_agents/adapters/__init__.py index 283b3c2..eacb35c 100644 --- a/src/verdi/adapters/__init__.py +++ b/src/verdi_agents/adapters/__init__.py @@ -2,6 +2,6 @@ These are OFF by default and lazily imported: importing this package never pulls in an optional dependency (e.g. ``gdm-concordia``) and never affects Verdi's -deterministic engine. See :mod:`verdi.adapters.concordia` for the Concordia +deterministic engine. See :mod:`verdi_agents.adapters.concordia` for the Concordia bridge and ``docs/concordia_eval.md`` for the integration assessment. """ diff --git a/src/verdi/adapters/concordia.py b/src/verdi_agents/adapters/concordia.py similarity index 99% rename from src/verdi/adapters/concordia.py rename to src/verdi_agents/adapters/concordia.py index eb446cc..d7071d4 100644 --- a/src/verdi/adapters/concordia.py +++ b/src/verdi_agents/adapters/concordia.py @@ -1,6 +1,6 @@ """Optional, OFF-by-default compatibility adapter to Google DeepMind Concordia. -This is a THIN bridge (todo D, follow-up). It exists so a Verdi agent / model +This is a THIN bridge. It exists so a Verdi agent / model endpoint can be surfaced in Concordia's `Entity` and `LanguageModel` interfaces for cross-framework inspection or co-simulation experiments. It is: diff --git a/src/verdi/adapters/concordia_reuse.py b/src/verdi_agents/adapters/concordia_reuse.py similarity index 99% rename from src/verdi/adapters/concordia_reuse.py rename to src/verdi_agents/adapters/concordia_reuse.py index cb247d3..fcc121d 100644 --- a/src/verdi/adapters/concordia_reuse.py +++ b/src/verdi_agents/adapters/concordia_reuse.py @@ -1,4 +1,4 @@ -"""Deep REUSE of Google DeepMind Concordia building blocks in Verdi (todo D). +"""Deep REUSE of Google DeepMind Concordia building blocks in Verdi. This module reuses Concordia's *actual* reusable components — not just its ABCs — for the parts where they genuinely reduce duplication in Verdi's auditing / diff --git a/src/verdi/agents/__init__.py b/src/verdi_agents/agents/__init__.py similarity index 100% rename from src/verdi/agents/__init__.py rename to src/verdi_agents/agents/__init__.py diff --git a/src/verdi/agents/client.py b/src/verdi_agents/agents/client.py similarity index 96% rename from src/verdi/agents/client.py rename to src/verdi_agents/agents/client.py index e7bd7f1..1064eb1 100644 --- a/src/verdi/agents/client.py +++ b/src/verdi_agents/agents/client.py @@ -1,6 +1,6 @@ """Async OpenAI-compatible client for Qwen on a vLLM endpoint. -Design rules (docs/ARCHITECTURE.md, GPU execution section): +Design rules: - bounded concurrency is enforced by the pool, not here; - every request carries a uuid trace id, returned on the result; - retries ONLY on transient 429/502/503 with capped exponential backoff and jitter; @@ -17,11 +17,11 @@ import httpx -from verdi.config import ModelEndpointConfig +from verdi_agents.config import ModelEndpointConfig # 524 is Cloudflare's origin-timeout (equivalent to 502/503: the upstream # origin — here a 27B reasoning model under load — took too long to answer). -# Stage C-1 attempt 6 died on an un-retried 524 (LESSONS Tranche 12g). +# A long live run once died on an un-retried 524, so it is retried too. RETRYABLE_STATUS = (429, 502, 503, 524) _BACKOFF_BASE = 0.5 _BACKOFF_CAP = 10.0 @@ -57,7 +57,7 @@ def _backoff_delay(attempt: int, rng: random.Random) -> float: class ModelClient: - """httpx-based client constructed from :class:`verdi.config.ModelEndpointConfig`.""" + """httpx-based client constructed from :class:`verdi_agents.config.ModelEndpointConfig`.""" def __init__( self, diff --git a/src/verdi/agents/policy.py b/src/verdi_agents/agents/policy.py similarity index 97% rename from src/verdi/agents/policy.py rename to src/verdi_agents/agents/policy.py index 2baa6ab..fdb20eb 100644 --- a/src/verdi/agents/policy.py +++ b/src/verdi_agents/agents/policy.py @@ -3,7 +3,7 @@ The policy calls the model, then parses and validates the JSON into the frozen ProposedAction schema. Malformed or schema-invalid output is *quarantined* (returned as (None, reason) and recorded), never raised: an LLM's invalid -completion must not crash the run (docs/ARCHITECTURE.md, GPU execution). +completion must not crash the run. """ from __future__ import annotations @@ -13,8 +13,8 @@ from pydantic import ValidationError -from verdi.agents.client import ModelResult -from verdi.schemas import ProposedAction, Task +from verdi_agents.agents.client import ModelResult +from verdi_agents.schemas import ProposedAction, Task #: Versioned prompt template constant. Bump deliberately; experiments log it. PROMPT_VERSION = "verdi.agent-policy-prompt/1" diff --git a/src/verdi/agents/pool.py b/src/verdi_agents/agents/pool.py similarity index 89% rename from src/verdi/agents/pool.py rename to src/verdi_agents/agents/pool.py index 6fecc76..9f7c395 100644 --- a/src/verdi/agents/pool.py +++ b/src/verdi_agents/agents/pool.py @@ -1,8 +1,7 @@ """Bounded async worker pool: dispatch policy calls, never exceed max_concurrency. Concurrency is enforced with an asyncio.Semaphore sized from the vLLM -throughput-derived cap in ModelEndpointConfig (docs/ARCHITECTURE.md, GPU -execution section). The pool never mutates world state; it only returns +throughput-derived cap in ModelEndpointConfig. The pool never mutates world state; it only returns per-task policy results. """ from __future__ import annotations @@ -11,8 +10,8 @@ from collections.abc import Awaitable, Callable from typing import Any -from verdi.agents.policy import PolicyResult -from verdi.schemas import Task +from verdi_agents.agents.policy import PolicyResult +from verdi_agents.schemas import Task class TaskPool: diff --git a/src/verdi/agents/recorded.py b/src/verdi_agents/agents/recorded.py similarity index 93% rename from src/verdi/agents/recorded.py rename to src/verdi_agents/agents/recorded.py index c42d930..f2e8727 100644 --- a/src/verdi/agents/recorded.py +++ b/src/verdi_agents/agents/recorded.py @@ -6,13 +6,13 @@ canned output, with the remaining tape consumed in order as fallback. This is what makes full-pipeline replay work without GPU -(docs/RESEARCH_SPEC.md: replayable snapshots without live model invocation). +(replayable snapshots without live model invocation). """ from __future__ import annotations import uuid -from verdi.agents.client import ModelResult +from verdi_agents.agents.client import ModelResult class TapeExhausted(RuntimeError): diff --git a/src/verdi/analysis/__init__.py b/src/verdi_agents/analysis/__init__.py similarity index 69% rename from src/verdi/analysis/__init__.py rename to src/verdi_agents/analysis/__init__.py index 1864850..8df88ec 100644 --- a/src/verdi/analysis/__init__.py +++ b/src/verdi_agents/analysis/__init__.py @@ -1,25 +1,25 @@ -"""Scenario loading, deterministic metric definitions and run reporting (Worker E, P1-E). +"""Scenario loading, deterministic metric definitions and run reporting. Public API ---------- -* :mod:`verdi.analysis.scenario` -- :class:`ScenarioLoader`, +* :mod:`verdi_agents.analysis.scenario` -- :class:`ScenarioLoader`, :func:`canonical_tape_hash`, :func:`build_t0_state`, :class:`LoadedScenario`. -* :mod:`verdi.analysis.metrics` -- pure, stdlib+pydantic-only metric +* :mod:`verdi_agents.analysis.metrics` -- pure, stdlib+pydantic-only metric functions (descriptive estimands; no LLM, no network). -* :mod:`verdi.analysis.report` -- :class:`RunReport` JSON/JSONL assembly plus - the G5 report path (:func:`build_g5_report`, :func:`write_g5_report`). -* :mod:`verdi.analysis.statistics` -- stdlib bootstrap/paired-effect +* :mod:`verdi_agents.analysis.report` -- :class:`RunReport` JSON/JSONL assembly plus + the paired-arm report (:func:`build_g5_report`, :func:`write_g5_report`). +* :mod:`verdi_agents.analysis.statistics` -- stdlib bootstrap/paired-effect estimand helpers (descriptive; never causal labels). -* :mod:`verdi.analysis.estimates` -- the six pre-registered estimands +* :mod:`verdi_agents.analysis.estimates` -- the six pre-registered estimands mapped to metric functions and matched-pair estimation. """ -from verdi.analysis.estimates import ( +from verdi_agents.analysis.estimates import ( ESTIMANDS, paired_estimates, run_estimand_values, run_estimates_for, ) -from verdi.analysis.metrics import ( +from verdi_agents.analysis.metrics import ( attempted_vs_executed_violations, false_sanction_rate, participation, @@ -28,7 +28,7 @@ value_drift_summary, welfare_and_inequality, ) -from verdi.analysis.report import ( +from verdi_agents.analysis.report import ( G5_ESTIMANDS, G5_LIMITATIONS, RunReport, @@ -36,7 +36,7 @@ build_g5_report, write_g5_report, ) -from verdi.analysis.scenario import ( +from verdi_agents.analysis.scenario import ( LoadedScenario, ScenarioLoader, ScenarioLoadError, @@ -44,7 +44,7 @@ build_t0_state, canonical_tape_hash, ) -from verdi.analysis.statistics import ( +from verdi_agents.analysis.statistics import ( PairedResult, bootstrap_ci, null_control_passes, diff --git a/src/verdi/analysis/comparative_report.py b/src/verdi_agents/analysis/comparative_report.py similarity index 98% rename from src/verdi/analysis/comparative_report.py rename to src/verdi_agents/analysis/comparative_report.py index fb94b13..256621a 100644 --- a/src/verdi/analysis/comparative_report.py +++ b/src/verdi_agents/analysis/comparative_report.py @@ -1,6 +1,6 @@ """Self-contained comparative HTML report for a paired fixed-vs-revisable run. -Renders :func:`verdi.analysis.outcome_metrics.summarize_pair` output as one +Renders :func:`verdi_agents.analysis.outcome_metrics.summarize_pair` output as one local, dependency-free HTML file (plain CSS tables; no external JS). The document is deterministic: sections, arms and seeds are sorted, values come only from the summary dict, and no generation timestamp is embedded. Every @@ -192,7 +192,7 @@ def _institution_rows(metrics: dict[str, Any]) -> list[str]: def _recovery_row(summary: dict[str, Any], arm: str, seed: Any) -> str: """Recovery section per (arm, seed) pair using the preregistered call.""" - from verdi.analysis.outcome_metrics import recovery_time + from verdi_agents.analysis.outcome_metrics import recovery_time metrics = summary["arms"][arm][str(seed)] try: diff --git a/src/verdi/analysis/cross_run_compare.py b/src/verdi_agents/analysis/cross_run_compare.py similarity index 100% rename from src/verdi/analysis/cross_run_compare.py rename to src/verdi_agents/analysis/cross_run_compare.py diff --git a/src/verdi/analysis/estimates.py b/src/verdi_agents/analysis/estimates.py similarity index 89% rename from src/verdi/analysis/estimates.py rename to src/verdi_agents/analysis/estimates.py index fe675ee..cf6921f 100644 --- a/src/verdi/analysis/estimates.py +++ b/src/verdi_agents/analysis/estimates.py @@ -1,17 +1,17 @@ -"""Wiring of the six pre-registered estimands to the metric helpers (P2 WS-4). +"""Compute the six pre-registered outcome estimands from a run and compare matched pairs. -This module is the bridge between the frozen estimand definitions in -``docs/PREREGISTRATION.md`` (v1.0) and the pure, deterministic metric -functions in :mod:`verdi.analysis.metrics` plus the descriptive -statistical helpers in :mod:`verdi.analysis.statistics`. +This module is the bridge between the frozen estimand definitions +(pre-registration v1.0) and the pure, deterministic metric +functions in :mod:`verdi_agents.analysis.metrics` plus the descriptive +statistical helpers in :mod:`verdi_agents.analysis.statistics`. -Estimand -> metric mapping (each PREREGISTRATION estimand is computed by -reusing an existing :mod:`verdi.analysis.metrics` function; nothing is +Estimand -> metric mapping (each preregistration estimand is computed by +reusing an existing :mod:`verdi_agents.analysis.metrics` function; nothing is re-implemented here):: estimand implementation (reuse-first) ----------------------- --------------------------------------------- - safety_rate ledger ratio: the PREREGISTRATION definition + safety_rate ledger ratio: the preregistration definition is "rejected / (accepted + rejected) over all attempted actions"; no metrics.py helper computes exactly that (the related @@ -21,7 +21,7 @@ directly from the action_accepted / action_rejected event kinds of the ledger false_punishment_rate false_sanction_rate - (OPERATIONAL definition, PREREGISTRATION + (OPERATIONAL definition, preregistration v1.1 addendum: appealed-and-upheld / all sanctions; sanctions reconstructed from the ledger's sanction_imposed events, @@ -32,7 +32,7 @@ inequality_gini_food welfare_and_inequality(states, "food") (Gini of agent-held food at the TERMINAL snapshot) - sustained_recovery OPERATIONAL definition (PREREGISTRATION + sustained_recovery OPERATIONAL definition (preregistration v1.1 addendum): the MEAN over evaluated treated agents of the fraction of post-restriction SNAPSHOT ticks at which @@ -46,7 +46,7 @@ (sum over sanctions of ticks the restriction was in force by run end) -Scientific honesty rules (binding, per PREREGISTRATION.md): +Scientific honesty rules (binding, per the preregistration): * **No causal labels**: nothing in this module names any quantity a "causal effect". The paired estimates report paired differences and, @@ -55,13 +55,13 @@ caller, not of the numbers. * **Single-run numbers are never population estimates**. ``paired_estimates`` is structured over a *sequence* of matched pairs: - with one pair per estimand, every :class:`~verdi.analysis.statistics + with one pair per estimand, every :class:`~verdi_agents.analysis.statistics .PairedResult` is returned with ``sufficient=False`` and ``ci_low`` / ``ci_high = None`` — the honest single-run state. Real bootstrap CIs appear only when ``n_pairs >= 10`` (the preregistered sufficiency threshold). -* ``states_by_tick`` is keyed by the LIFECYCLE tick (ADR-008) — the - pilot's snapshot map is taken as-is; the reducer's cumulative +* ``states_by_tick`` is keyed by the LIFECYCLE tick — the + reference run's snapshot map is taken as-is; the reducer's cumulative transition counter is not a valid key. * Missing evidence stays ``None``... but the six estimands are defined to be floats: where the preregistration defines a ratio with a zero @@ -70,7 +70,7 @@ supporting counts exposed by the per-call ``detail`` returned from :func:`_sustained_recovery_detail` / the metric functions themselves. -Implementation is stdlib + pydantic (schemas) only, per ADR-009; no RNG +Implementation is stdlib + pydantic (schemas) only; no RNG lives in this module (all randomness is inside the seeded statistics helpers), so outputs are deterministic for identical inputs. """ @@ -79,14 +79,14 @@ from collections.abc import Mapping, Sequence from typing import Any -from verdi.analysis.metrics import ( +from verdi_agents.analysis.metrics import ( false_sanction_rate, sustained_recovery, time_under_deprivation, welfare_and_inequality, ) -from verdi.analysis.statistics import PairedResult, paired_effect -from verdi.schemas import Appeal, EvidenceReference, Sanction, WorldEvent, WorldState +from verdi_agents.analysis.statistics import PairedResult, paired_effect +from verdi_agents.schemas import Appeal, EvidenceReference, Sanction, WorldEvent, WorldState __all__ = [ "ESTIMANDS", @@ -107,13 +107,13 @@ ) #: Pre-specified consecutive-tick count for the k-consecutive-tick -#: criterion of :func:`verdi.analysis.metrics.sustained_recovery` -#: (PREREGISTRATION estimand 5: "sustained ... not just instant"). One +#: criterion of :func:`verdi_agents.analysis.metrics.sustained_recovery` +#: (preregistration estimand 5: "sustained ... not just instant"). One #: consecutive tick after the restriction ends counts for this frozen -#: threshold; the pilot's snapshot cadence (every ~5-7 lifecycle +#: threshold; the reference run's snapshot cadence (every ~5-7 lifecycle #: ticks) means a larger k would be unobservable. The ESTIMAND value #: itself is the per-agent snapshot-fraction mean declared by the -#: PREREGISTRATION v1.1 addendum; ``k`` only drives the per-agent +#: preregistration v1.1 addendum; ``k`` only drives the per-agent #: ``sustained`` boolean kept in the detail output. SUSTAINED_RECOVERY_K = 1 @@ -127,12 +127,12 @@ # --------------------------------------------------------------------------- # def _payload_tick(event: WorldEvent) -> int: - """LIFECYCLE tick recorded in an event payload (ADR-008).""" + """LIFECYCLE tick recorded in an event payload.""" return int(event.payload.get("tick", 0)) def _reconstruct_sanctions(events: Sequence[WorldEvent]) -> list[Sanction]: - """Rebuild :class:`~verdi.schemas.Sanction` objects from ledger events. + """Rebuild :class:`~verdi_agents.schemas.Sanction` objects from ledger events. A ``sanction_imposed`` event with ``decision='accepted'`` carries ``sanction_id``, ``subject_id``, ``scope``, ``duration_ticks`` and @@ -173,7 +173,7 @@ def _reconstruct_sanctions(events: Sequence[WorldEvent]) -> list[Sanction]: def _reconstruct_appeal_outcomes(events: Sequence[WorldEvent]) -> list[Appeal]: - """Rebuild :class:`~verdi.schemas.Appeal` objects with final status. + """Rebuild :class:`~verdi_agents.schemas.Appeal` objects with final status. An ``appeal_filed`` (accepted) event starts a record; a later ``appeal_upheld`` (accepted) or ``appeal_dismissed`` event sets the @@ -246,14 +246,14 @@ def _sustained_recovery_detail( agent's held resource is >= its pre-sanction level, divided by the number of post-restriction ticks (0.0 when there is no post-restriction snapshot evidence). - * The :func:`verdi.analysis.metrics.sustained_recovery` + * The :func:`verdi_agents.analysis.metrics.sustained_recovery` k-consecutive-tick boolean is also computed per agent (``k`` consecutive post-restriction snapshot ticks meeting the level) and kept in the detail as ``sustained``. * The ESTIMAND value is the MEAN of the per-agent fractions over all TREATED agents whose restriction has ended by the terminal tick (0.0 if none evaluated). This is the OPERATIONAL definition - declared by the PREREGISTRATION v1.1 addendum for estimand 5 + declared by the preregistration v1.1 addendum for estimand 5 (``sustained_recovery``): a true per-lifecycle-tick fraction is unobservable because world snapshots exist only at the run's snapshot cadence, not at every tick after the treatment. @@ -311,7 +311,7 @@ def _sustained_recovery_detail( terminal = ticks_sorted[-1] if ticks_sorted else 0 evaluated = [a for a, d in per_agent.items() if d["restriction_end_tick"] <= terminal] recovered = [a for a in evaluated if per_agent[a]["sustained"]] - # ESTIMAND value (PREREGISTRATION v1.1 addendum, operational + # ESTIMAND value (preregistration v1.1 addendum, operational # definition for estimand 5): the MEAN of the per-agent # post-restriction snapshot-tick fractions over the evaluated # treated agents. 0.0 when no agent is evaluated (no evidence). @@ -339,12 +339,12 @@ def run_estimand_values( """Compute the six pre-registered estimand values for one run arm. ``states_by_tick`` maps the LIFECYCLE tick of each snapshot to the - world state (ADR-008) — pass the pilot's ``state_snapshots`` as-is - (``{t: st for t, st in pilot.state_snapshots}``). ``log`` is the - immutable event ledger (``pilot.log.events`` or an EventLog). + world state — pass the reference run's ``state_snapshots`` as-is + (``{t: st for t, st in run.state_snapshots}``). ``log`` is the + immutable event ledger (``run.log.events`` or an EventLog). Every estimand is computed by reusing a - :mod:`verdi.analysis.metrics` function (see the module docstring + :mod:`verdi_agents.analysis.metrics` function (see the module docstring mapping). The terminal snapshot for the welfare/inequality estimands is the maximum lifecycle tick present in ``states_by_tick``. @@ -369,7 +369,7 @@ def run_estimand_values( raise ValueError("states_by_tick must contain at least one snapshot") # 1. safety_rate — rejected / (accepted + rejected) over all attempted - # actions (the exact PREREGISTRATION ratio). + # actions (the exact preregistration ratio). action_events = [e for e in events if e.kind in _ACTION_KINDS] accepted_actions = sum(1 for e in action_events if e.kind == "action_accepted") rejected_actions = sum(1 for e in action_events if e.kind == "action_rejected") @@ -466,17 +466,17 @@ def paired_estimates( * two estimand-value dicts (mappings with the six :data:`ESTIMANDS` keys). - Pairs must be matched per PREREGISTRATION.md (same sealed snapshot, + Pairs must be matched per the preregistration (same sealed snapshot, same remaining task tape); matching is the *caller's* responsibility — this function only consumes values. **Honesty rule**: with a single pair (``n_pairs == 1``) every - :class:`~verdi.analysis.statistics.PairedResult` is returned with - ``sufficient=False`` and ``ci_low = ci_high = None`` — one pilot + :class:`~verdi_agents.analysis.statistics.PairedResult` is returned with + ``sufficient=False`` and ``ci_low = ci_high = None`` — one run per arm is never a population estimate, and the bootstrap is not applicable to a single difference. With ``n_pairs >= 2`` the per-estimand paired differences are passed to - :func:`verdi.analysis.statistics.paired_effect`, which reports + :func:`verdi_agents.analysis.statistics.paired_effect`, which reports real bootstrap CIs only from ``n_pairs >= 10`` (the preregistered sufficiency threshold); 2-9 pairs report the point estimate flagged insufficient. @@ -509,7 +509,7 @@ def paired_estimates( results: dict[str, PairedResult] = {} if len(normalized) < 2: - # The honest single-run state (PREREGISTRATION "Uncertainty"): + # The honest single-run state (preregistration "Uncertainty"): # report the estimate, withhold the CI, flag insufficient. for name in ESTIMANDS: diffs = [float(t[name]) - float(p[name]) for t, p in normalized] diff --git a/src/verdi/analysis/html_report.py b/src/verdi_agents/analysis/html_report.py similarity index 99% rename from src/verdi/analysis/html_report.py rename to src/verdi_agents/analysis/html_report.py index e7d886b..07785d8 100644 --- a/src/verdi/analysis/html_report.py +++ b/src/verdi_agents/analysis/html_report.py @@ -1,4 +1,4 @@ -"""Standalone, read-only society report: python -m verdi.analysis.html_report RUN_DIR. +"""Standalone, read-only society report: python -m verdi_agents.analysis.html_report RUN_DIR. Chart/UI libraries are MIT licensed; their pinned distributions retain license headers. Default output embeds cached/downloaded assets. --cdn explicitly requires connectivity. @@ -101,7 +101,7 @@ def write_report(run_dir: Path, output: Path, *, cdn: bool = False, f'') data['warnings'].append('CDN mode: internet required to load chart and UI libraries') else: - cache = asset_dir or Path.home() / '.cache' / 'verdi' / 'html-report' + cache = asset_dir or Path.home() / '.cache' / 'verdi-agents' / 'html-report' cache.mkdir(parents=True, exist_ok=True) contents = {} for name, url in ASSETS.items(): diff --git a/src/verdi/analysis/metrics.py b/src/verdi_agents/analysis/metrics.py similarity index 98% rename from src/verdi/analysis/metrics.py rename to src/verdi_agents/analysis/metrics.py index 0d822a4..eae75aa 100644 --- a/src/verdi/analysis/metrics.py +++ b/src/verdi_agents/analysis/metrics.py @@ -1,4 +1,4 @@ -"""Deterministic metric definitions for Verdi runs (P1-E). +"""Deterministic metric definitions for Verdi runs. Design rules ------------ @@ -8,7 +8,7 @@ that has no evidence must not look like a metric that measured nothing. * Every docstring states the estimand and whether it is *descriptive* (a property of the observed run) or *interventional* (it only becomes a - causal contrast when run under matched experimental arms G0-G4). + causal contrast when run under matched experimental arms). The declared/private/public probe channels are always kept separate: combining them would conflate self-report with observed behavior, which @@ -19,7 +19,7 @@ from collections.abc import Mapping, Sequence from typing import Any -from verdi.schemas import ProbeResult, Sanction, WorldEvent, WorldState +from verdi_agents.schemas import ProbeResult, Sanction, WorldEvent, WorldState __all__ = [ "attempted_vs_executed_violations", diff --git a/src/verdi/analysis/outcome_metrics.py b/src/verdi_agents/analysis/outcome_metrics.py similarity index 98% rename from src/verdi/analysis/outcome_metrics.py rename to src/verdi_agents/analysis/outcome_metrics.py index a25672c..2918cb3 100644 --- a/src/verdi/analysis/outcome_metrics.py +++ b/src/verdi_agents/analysis/outcome_metrics.py @@ -1,4 +1,4 @@ -"""P0-D primary outcome metrics extracted from recorded run artifacts. +"""Primary outcome metrics extracted from recorded run artifacts. Design rules ------------ @@ -23,7 +23,7 @@ * ``exclusion_episodes``-- sanction episodes per agent, excluded ticks, repeat-episode flags. "Excluded from" = the transfer+deposit capability withdrawn under sanction (``SANCTION_BLOCKED_KINDS`` in - :mod:`verdi.world.validate`); the world-state flag (rights_change tape + :mod:`verdi_agents.world.validate`); the world-state flag (rights_change tape steps) defines when it is in force. * ``recovery_time`` -- preregistration-style: the caller supplies the pre-shock baseline window, post-shock start, target fraction and @@ -44,7 +44,7 @@ from typing import Any #: Action kinds whose execution is withdrawn while an agent is under sanction -#: (mirrors verdi.world.validate.SANCTION_BLOCKED_KINDS; re-declared here to +#: (mirrors verdi_agents.world.validate.SANCTION_BLOCKED_KINDS; re-declared here to #: keep this module dependency-light for offline artifact analysis). SANCTION_BLOCKED_KINDS = frozenset({"transfer", "deposit"}) @@ -151,7 +151,7 @@ def _declared_due_by_objective(run_dir: Path) -> dict[str, int]: try: import yaml - from verdi.world.economy import parse_task_objectives + from verdi_agents.world.economy import parse_task_objectives resolved = yaml.safe_load( (run_dir / "scenario_resolved.yaml").read_text(encoding="utf-8")) @@ -169,7 +169,7 @@ def completion_rate(run_dir: Path) -> dict[str, Any]: Each delivery objective is evaluated by the campaign from committed world state at its due tick and ledgered as exactly one ``task_completed`` / ``task_failed`` event whose payload carries the full - :func:`verdi.world.economy.check_delivery` outcome. This function only + :func:`verdi_agents.world.economy.check_delivery` outcome. This function only re-reads those ledger events — it trusts no self-report. Overall denominator (explicit): the number of delivery objectives @@ -348,7 +348,7 @@ def exclusion_episodes(run_dir: Path) -> dict[str, Any]: "Excluded from" is defined explicitly: while an agent is under sanction the validator blocks the ``transfer`` and ``deposit`` action kinds - (``verdi.world.validate.SANCTION_BLOCKED_KINDS``); the episode spans the + (``verdi_agents.world.validate.SANCTION_BLOCKED_KINDS``); the episode spans the sanction's ``effective_tick`` until restoration (duration expiry or an upheld appeal, whichever lands first in ``sanctions.jsonl``) or run end (restored_tick = null, still excluded at the last observed tick). @@ -773,7 +773,7 @@ def _load_pair_manifest(output_root: Path, manifest_path: Path | None) -> dict[s if not path.exists(): raise RunArtifactError( f"pair manifest not found: {path} — run the paired runner first " - "(python -m verdi.experiments.paired_run)" + "(python -m verdi_agents.experiments.paired_run)" ) try: manifest = json.loads(path.read_text(encoding="utf-8")) @@ -787,7 +787,7 @@ def _load_pair_manifest(output_root: Path, manifest_path: Path | None) -> dict[s def summarize_pair(output_root: Path, manifest_path: Path | None = None) -> dict[str, Any]: """Compare every arm of a paired experiment from its on-disk artifacts. - Reads ``pair_manifest.json`` (from the P0-C paired runner) plus each arm's + Reads ``pair_manifest.json`` (from the paired runner) plus each arm's run dir. Arms whose manifest status is not ``complete`` are listed under ``skipped`` with the recorded reason instead of being silently dropped (partial failure stays visible in the comparison). @@ -959,7 +959,7 @@ def _csv_cell(value: Any) -> str: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - description="Extract P0-D primary outcome metrics from a paired run root." + description="Extract primary outcome metrics from a paired run root." ) parser.add_argument("--pair-root", required=True, type=Path, help="output root written by the paired runner " @@ -982,7 +982,7 @@ def main(argv: list[str] | None = None) -> int: root = args.pair_root json_path = write_metrics_json(summary, root / "metrics.json") csv_path = write_metrics_csv(summary, root / "metrics.csv") - from verdi.analysis.comparative_report import write_comparison_html + from verdi_agents.analysis.comparative_report import write_comparison_html html_path = write_comparison_html(summary, root / "comparison.html") print(f"wrote {json_path}") print(f"wrote {csv_path}") diff --git a/src/verdi/analysis/report.py b/src/verdi_agents/analysis/report.py similarity index 90% rename from src/verdi/analysis/report.py rename to src/verdi_agents/analysis/report.py index 497e973..8ccfe2b 100644 --- a/src/verdi/analysis/report.py +++ b/src/verdi_agents/analysis/report.py @@ -1,4 +1,4 @@ -"""Run report assembly for Verdi experiments (P1-E). +"""Run report assembly for Verdi experiments. A :class:`RunReport` is a plain, JSON-serializable record of one run: manifest, lifecycle ticks, computed metrics, event count, final state @@ -8,7 +8,7 @@ run cannot support (demo scale, single-seed, descriptive-only, ...). * Missing data appears as ``null`` / absent keys, never as zero. * No findings are invented: only values computed by - :mod:`verdi.analysis.metrics` or explicitly supplied by the caller + :mod:`verdi_agents.analysis.metrics` or explicitly supplied by the caller appear in ``metrics``. """ from __future__ import annotations @@ -44,7 +44,7 @@ class RunReport: actually-observed per-tick record. metrics: Mapping of metric name -> dict produced by - :mod:`verdi.analysis.metrics` (or explicitly null for + :mod:`verdi_agents.analysis.metrics` (or explicitly null for unmeasured metrics). event_count: Number of immutable events appended, or None if unknown. @@ -123,10 +123,10 @@ def write(self, path: str | Path) -> Path: # --------------------------------------------------------------------------- # -# G5 statistical sanity report (P2 WS-4) +# Paired-arm statistical report # --------------------------------------------------------------------------- # -#: The six frozen primary estimands (docs/PREREGISTRATION.md v1.0). +#: The six frozen primary estimands (pre-registration v1.0). G5_ESTIMANDS = ( "safety_rate", "false_punishment_rate", @@ -136,7 +136,7 @@ def write(self, path: str | Path) -> Path: "time_under_deprivation", ) -#: Limitations every G5 report MUST carry (deterministic, in this order). +#: Limitations every paired-arm report MUST carry (deterministic, in this order). G5_LIMITATIONS: tuple[str, ...] = ( "recorded-output pilot: no live LLM interaction was measured; agent " "behavior is a deterministic stand-in", @@ -150,7 +150,7 @@ def write(self, path: str | Path) -> Path: def _canonical_state_hash(state) -> str: - from verdi.world.reducer import WorldReducer + from verdi_agents.world.reducer import WorldReducer return WorldReducer.canonical_state_hash(state) @@ -160,7 +160,7 @@ def _arm_trajectory( estimates: dict[str, float] | None, ) -> dict[str, Any]: """Per-arm raw trajectory: per LIFECYCLE-tick canonical state hashes - (ADR-008) plus the arm's terminal estimand values.""" + plus the arm's terminal estimand values.""" ticks: dict[str, str] = {} for tick, state in sorted(snapshots or [], key=lambda item: int(item[0])): ticks[str(tick)] = _canonical_state_hash(state) @@ -173,7 +173,7 @@ def _null_control(pairs_results: dict[str, Any]) -> dict[str, Any]: The same run's estimand values vs the same run's values produce exactly-zero paired differences (the arm is measurably inert against itself); the differences are checked with - :func:`verdi.analysis.statistics.null_control_passes` (bootstrap CI + :func:`verdi_agents.analysis.statistics.null_control_passes` (bootstrap CI of the mean difference contains 0) and by direct comparison to 0. ``n_pairs`` is reported honestly (a self-comparison of one run is not a replication design — it is a wiring/data-integrity check). @@ -188,7 +188,7 @@ def _null_control(pairs_results: dict[str, Any]) -> dict[str, Any]: an explicit ``ci_input: "insufficient pairs"`` marker rather than a vacuous computation over constant zeros. """ - from verdi.analysis.statistics import null_control_passes + from verdi_agents.analysis.statistics import null_control_passes per_estimand: dict[str, Any] = {} differences: list[float] = [] @@ -225,10 +225,10 @@ def _null_control(pairs_results: dict[str, Any]) -> dict[str, Any]: } -#: Required top-level keys for a WS-2 arm run dict. ``estimates`` is +#: Required top-level keys for an arm run dict. ``estimates`` is #: required unless the run dict instead carries both ``states_by_tick`` #: and ``log`` (from which estimands can be computed), in which case the -#: estimator falls back to :func:`verdi.analysis.estimates.run_estimates_for`. +#: estimator falls back to :func:`verdi_agents.analysis.estimates.run_estimates_for`. _REQUIRED_ARM_KEYS: tuple[str, ...] = ( "arm", "applied", @@ -244,10 +244,10 @@ def _null_control(pairs_results: dict[str, Any]) -> dict[str, Any]: def _validate_arm_run_dict( arm_name: str, run: Mapping[str, Any] ) -> None: - """Validate the required key set for a WS-2 arm run dict up front. + """Validate the required key set for an arm run dict up front. Raises :class:`ValueError` naming every missing required key so a - mis-constructed run dict never silently yields nulls in the G5 + mis-constructed run dict never silently yields nulls in the report. ``estimates`` (a mapping with the six estimand keys) is required unless BOTH ``states_by_tick`` and ``log`` are present, in which case the estimands can be recomputed from the raw material. @@ -285,18 +285,18 @@ def build_g5_report( placebo_run: dict[str, Any], pairs_results: dict[str, Any], ) -> dict[str, Any]: - """Assemble the schema-free, JSON-serializable G5 report. + """Assemble the schema-free, JSON-serializable paired-arm report. - ``pilot`` is a :class:`~verdi.pilot.driver.PilotRun` (or any object + ``pilot`` is a :class:`~verdi_agents.reference_run.driver.PilotRun` (or any object with ``manifest`` / ``final_state`` / ``state_snapshots`` - attributes). ``treatment_run`` / ``placebo_run`` are WS-2 run dicts + attributes). ``treatment_run`` / ``placebo_run`` are arm run dicts (``arm``, ``applied``, ``reason``, ``manifest``, ``state_snapshots``, ``estimates``); each arm's ``estimates`` are produced by - :func:`verdi.analysis.estimates.run_estimand_values` and both arms + :func:`verdi_agents.analysis.estimates.run_estimand_values` and both arms must fold to the same final hash when inert (``applied=False``). ``pairs_results`` is the per-estimand - :class:`~verdi.analysis.statistics.PairedResult` mapping from - :func:`verdi.analysis.estimates.paired_estimates`. + :class:`~verdi_agents.analysis.statistics.PairedResult` mapping from + :func:`verdi_agents.analysis.estimates.paired_estimates`. Both run dicts are validated up front (:func:`_validate_arm_run_dict`): ``arm``, ``applied``, ``reason``, ``final_state_hash``, @@ -304,7 +304,7 @@ def build_g5_report( ``states_by_tick`` + ``log``); missing keys raise :class:`ValueError` rather than silently producing nulls in the report. - Contents (G5, docs/ACCEPTANCE.md): raw per-arm trajectories + Contents: raw per-arm trajectories (per-lifecycle-tick canonical state hashes + estimand values), per-estimand paired results with CI-or-null and the sufficiency flag, the null control (placebo vs itself), and the required @@ -363,7 +363,7 @@ def build_g5_report( def write_g5_report(report: dict[str, Any], path: str | Path) -> Path: - """Write the G5 report as deterministic JSON. + """Write the paired-arm report as deterministic JSON. Pretty-printed, ``sort_keys=True``, trailing newline: for identical report dicts the bytes are identical (stable across repeated diff --git a/src/verdi/analysis/scenario.py b/src/verdi_agents/analysis/scenario.py similarity index 96% rename from src/verdi/analysis/scenario.py rename to src/verdi_agents/analysis/scenario.py index 760e718..6fd12b9 100644 --- a/src/verdi/analysis/scenario.py +++ b/src/verdi_agents/analysis/scenario.py @@ -1,7 +1,7 @@ """Deterministic scenario + task-tape loading into frozen v1 schemas. The scenario JSON describes the world at tick 0 (agents, resources, -versions, seed) and the pilot lifecycle map. The task tape is a separate, +versions, seed) and the lifecycle map. The task tape is a separate, content-addressed file: its ``tape_hash`` is the sha256 of the canonical (sorted-key, compact) JSON of the tape *excluding* the hash field itself, and is what ``ExperimentManifest.task_tape_hash`` must carry. @@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, Field -from verdi.schemas import Task, WorldState +from verdi_agents.schemas import Task, WorldState __all__ = [ "LoadedScenario", @@ -33,7 +33,7 @@ def load_scenario(path: str | Path) -> dict[str, Any]: """Load a scenario definition from a ``.json`` or ``.yml``/``.yaml`` file. - Tranche 12l: YAML is a superset of JSON, so a single authoring format + YAML is a superset of JSON, so a single authoring format works for humans (comments, no braces) while JSON stays fully supported for existing scenario files. ``.json`` files are parsed with the stdlib JSON parser (no behavior change); ``.yml``/``.yaml`` use PyYAML. @@ -97,7 +97,7 @@ def canonical_tape_hash(payload: dict[str, Any]) -> str: def build_t0_state(scenario: dict[str, Any]) -> WorldState: """Build the frozen tick-0 WorldState from a scenario dict. - Tranche 12k: the scenario is the single source of truth for the world's + The scenario is the single source of truth for the world's rules. Optional ``constitution`` and ``law`` blocks are normalized into the returned state (with defaults identical to the historical hardcoded values), so a scenario file fully defines the simulation it drives. @@ -109,7 +109,7 @@ def build_t0_state(scenario: dict[str, Any]) -> WorldState: raise ScenarioLoadError("scenario contains duplicate agent ids") _check_values(scenario, set(agent_ids)) _normalize_governance(scenario) - # Tranche 12m: an agent entry may carry a `sanction: {duration_ticks: N}` + # An agent entry may carry a `sanction: {duration_ticks: N}` # block (a scenario-declared t0 sanction with auto-expiry). That is campaign # metadata, not frozen world state, so strip it before building WorldState. # The campaign reads it from the scenario dict directly. @@ -171,7 +171,7 @@ def _normalize_governance(scenario: dict[str, Any]) -> None: Defaults reproduce the historical hardcoded values exactly, so existing scenario files (without these blocks) behave byte-identically. The - normalized blocks are what :class:`~verdi.society.campaign.SocietyCampaign` + normalized blocks are what :class:`~verdi_agents.society.campaign.SocietyCampaign` builds its GovernanceEngine from — one file, one definition. """ constitution = scenario.get("constitution") or {} @@ -207,7 +207,7 @@ def _normalize_economy(scenario: dict[str, Any]) -> None: from the scenario dict — this function only validates, it does not add keys, so existing scenarios' normalized dicts are unchanged. """ - from verdi.world.economy import parse_economy_spec, parse_task_objectives + from verdi_agents.world.economy import parse_economy_spec, parse_task_objectives if "economy" in scenario and scenario["economy"]: parse_economy_spec(scenario["economy"]) # raises on malformed diff --git a/src/verdi/analysis/statistics.py b/src/verdi_agents/analysis/statistics.py similarity index 93% rename from src/verdi/analysis/statistics.py rename to src/verdi_agents/analysis/statistics.py index 421c0c7..41229a9 100644 --- a/src/verdi/analysis/statistics.py +++ b/src/verdi_agents/analysis/statistics.py @@ -1,7 +1,6 @@ -"""G5 descriptive estimation helpers (P2 WS-3, frozen per docs/P2_CONTRACT.md). +"""Descriptive statistics (bootstrap CIs, paired differences) for paired experiments. -These are **descriptive** estimation helpers for pre-registered estimands -(docs/PREREGISTRATION.md v1.0): +These are **descriptive** estimation helpers for the pre-registered estimands: * :func:`bootstrap_ci` — nonparametric percentile bootstrap confidence interval of the MEAN, deterministic for a given ``seed``. @@ -12,7 +11,7 @@ Statistical honesty rules (binding): -* Per PREREGISTRATION.md ("No causal label without randomization"), these +* No causal label without randomization: these helpers NEVER label effects causal. They report a point estimate and an uncertainty interval for whatever differences/quantities are passed in; interpretation of the design belongs to the caller. @@ -23,7 +22,7 @@ coverage. Implementation is stdlib only (``random``, ``math``, ``statistics``) — no -numpy/scipy, no third-party imports, per ADR-009. +numpy/scipy, no third-party imports. """ from __future__ import annotations @@ -59,7 +58,7 @@ def bootstrap_ci( using ``random.Random(seed)`` and returns the percentile interval ``((1 - ci) / 2, 1 - (1 - ci) / 2)`` over the bootstrap means. - Percentile convention (P2 review F4): the bounds are the *inclusive* + Percentile convention: the bounds are the *inclusive* linear-interpolation percentiles of the sorted bootstrap means, i.e. exactly the values ``statistics.quantiles(means, n=200, method="inclusive")`` would return at indices @@ -147,7 +146,7 @@ def paired_effect( """Estimate a paired treatment-vs-placebo effect (descriptive, not causal). The paired differences are ``treatment[i] - placebo[i]`` for matched - pairs (matched snapshots + matched task tape per PREREGISTRATION.md). + pairs (matched snapshots + matched task tape per the preregistration). ``estimate`` is the mean paired difference. When at least 10 pairs are available (``sufficient``), a percentile bootstrap CI of the mean difference is reported using the same ``seed``; otherwise the CI is @@ -202,7 +201,7 @@ def null_control_passes( ) -> bool: """Negative-control check: True iff the bootstrap CI of the mean contains 0. - Used for null-drift / no-intervention arms per PREREGISTRATION.md + Used for null-drift / no-intervention arms per the preregistration ("Negative controls: null drift ... probe deltas CI contains 0 on a no-intervention arm"). """ diff --git a/src/verdi/audit/__init__.py b/src/verdi_agents/audit/__init__.py similarity index 53% rename from src/verdi/audit/__init__.py rename to src/verdi_agents/audit/__init__.py index 15ea7f8..a232079 100644 --- a/src/verdi/audit/__init__.py +++ b/src/verdi_agents/audit/__init__.py @@ -1,5 +1,5 @@ """Adapter-level audit evidence: TraceSource protocol, AuditAdapter, sources.""" -from verdi.audit.adapter import AuditAdapter, TraceSource -from verdi.audit.in_memory import InMemoryTraceSource +from verdi_agents.audit.adapter import AuditAdapter, TraceSource +from verdi_agents.audit.in_memory import InMemoryTraceSource __all__ = ["AuditAdapter", "InMemoryTraceSource", "TraceSource"] diff --git a/src/verdi/audit/adapter.py b/src/verdi_agents/audit/adapter.py similarity index 96% rename from src/verdi/audit/adapter.py rename to src/verdi_agents/audit/adapter.py index 33249db..a477313 100644 --- a/src/verdi/audit/adapter.py +++ b/src/verdi_agents/audit/adapter.py @@ -4,7 +4,7 @@ ``TraceSource`` protocol that mirrors SimpleAudit's ``TraceProvider`` contract (``start()`` / ``endpoint`` / ``fetch(trace_id)`` / ``stop()``). Any in-memory fake or the real SimpleAudit provider (via -:mod:`verdi.audit.simpleaudit_bridge`) satisfies it. +:mod:`verdi_agents.audit.simpleaudit_bridge`) satisfies it. Guarantees: - Records never embed raw span attributes, only span identity and names, @@ -17,7 +17,7 @@ from typing import Any, Protocol, runtime_checkable -from verdi.schemas import AuditRecord, WorldEvent +from verdi_agents.schemas import AuditRecord, WorldEvent @runtime_checkable diff --git a/src/verdi/audit/in_memory.py b/src/verdi_agents/audit/in_memory.py similarity index 100% rename from src/verdi/audit/in_memory.py rename to src/verdi_agents/audit/in_memory.py diff --git a/src/verdi/audit/interview_audit.py b/src/verdi_agents/audit/interview_audit.py similarity index 99% rename from src/verdi/audit/interview_audit.py rename to src/verdi_agents/audit/interview_audit.py index e67770b..e334811 100644 --- a/src/verdi/audit/interview_audit.py +++ b/src/verdi_agents/audit/interview_audit.py @@ -1,8 +1,8 @@ """SimpleAudit evidence-anchored checklist judgments over interview answers. -This is the REAL SimpleAudit integration lane (todo B). It reuses the installed +This is the real SimpleAudit integration for interviews. It reuses the installed ``simpleaudit`` package (v0.4.x) — it does NOT reimplement an auditing -framework. The pattern mirrors :mod:`verdi.audit.judge_live` (ADR-012): +framework. The pattern mirrors :mod:`verdi_agents.audit.judge`: * a SEALED target returns the agent's own verbatim interview answer, so the judge model grades fixed text (deterministic-replayable, no extra target @@ -29,7 +29,7 @@ from simpleaudit.model_auditor import ModelAuditor from simpleaudit.targets import TargetContext, TargetResponse -from verdi.audit.judge_live import _assert_no_key_material +from verdi_agents.audit.judge import _assert_no_key_material #: Value axes the checklist reasons about (fixed order). VALUE_AXES = ("integrity", "fairness", "cooperation", "autonomy", "solidarity") @@ -97,7 +97,7 @@ class SealedInterviewTarget: Returns the VERBATIM captured interview answer as the single turn. No network, no target-model involvement — exactly like ``SealedTapeTarget`` in - :mod:`verdi.audit.judge_live`. + :mod:`verdi_agents.audit.judge`. """ def __init__(self, replay: str) -> None: diff --git a/src/verdi/audit/judge_live.py b/src/verdi_agents/audit/judge.py similarity index 95% rename from src/verdi/audit/judge_live.py rename to src/verdi_agents/audit/judge.py index 7b109ef..e5ad273 100644 --- a/src/verdi/audit/judge_live.py +++ b/src/verdi_agents/audit/judge.py @@ -1,24 +1,24 @@ -"""ADR-012: independent LIVE judge-model pass over the L1 live tape (G4). +"""Ask a live judge model to grade each recorded model call of a captured live run. -Adds the missing G4 piece from the L2 bridge (``live_bridge.py``): a judge +Extends the trace audit (``trace_audit.py``) with a judge **model** verdict per sampled dispatch row, reported against the deterministic code-level judgment (SimpleAudit ``grade_choice_match``) and the engine decision. Correlation reuses -:func:`~verdi.audit.live_bridge.correlate_tape_to_ledger` — sealed +:func:`~verdi_agents.audit.trace_audit.correlate_tape_to_ledger` — sealed ``event_id`` only, never positional. SimpleAudit usage (recorded in the report's ``simpleaudit_usage`` section): -- ``ModelAuditor`` is used through its PUBLIC API exactly as ADR-012 §A - prescribes: ``ModelAuditor(model=..., provider="openai", judge_model=..., +- ``ModelAuditor`` is used through its PUBLIC API only: + ``ModelAuditor(model=..., provider="openai", judge_model=..., judge_provider="openai", api_key=..., base_url=..., judge_api_key=..., judge_base_url=...)`` with ``judge_prompt``/``judge_response_schema``, then ``auditor.set_target(SealedTapeTarget(...))`` and ``auditor.run(scenarios, max_turns=1, max_workers=1)``. - ONE adapter (documented in the report): ``SealedTapeTarget``, a - ``simpleaudit.targets.Target`` implementation. The L1 transcript is - sealed (re-running the target model would re-burn budget and break the - ADR-010 replay contract), so the target's single turn replays the + ``simpleaudit.targets.Target`` implementation. The captured transcript is + sealed (re-running the target model would re-burn budget and break + replay-from-tape), so the target's single turn replays the CAPTURED assistant response verbatim; ``ModelAuditor.run_scenario`` then makes exactly ONE live judge call over that transcript. ``ScenarioExecution`` is produced by ModelAuditor with the sealed ``event_id`` correlation ids @@ -29,13 +29,13 @@ enforced: SimpleAudit's internal retry loop is otherwise invisible to this module's call counter. -Framing disclosure (A1 addendum, B2): the judge is SHOWN the engine +Framing disclosure: the judge is SHOWN the engine outcome in its prompt. The measured quantity is therefore judge-to-engine -SELF-CONSISTENCY, not an independent compliance judgment. G4's blinding -clause is satisfied by the CODE judge (the label reaches only the +SELF-CONSISTENCY, not an independent compliance judgment. The blinding +requirement is satisfied by the CODE judge (the label reaches only the grader via ``decision.accepted``), not by the model judge. -Judge nondeterminism (ADR-012 §C): the judge endpoint is acknowledged +Judge nondeterminism: the judge endpoint is acknowledged nondeterministic. The run is reproducible FROM the recorded judge tape (full judge prompt + response + usage captured verbatim under ``runs/live/judge//``), not bit-reproducible on re-run. @@ -57,7 +57,7 @@ from simpleaudit.model_auditor import ModelAuditor from simpleaudit.targets import TargetContext, TargetResponse -from verdi.audit.live_bridge import ( +from verdi_agents.audit.trace_audit import ( LiveLedgerError, _import_simpleaudit, _model_choice, @@ -69,20 +69,20 @@ load_ledger_events, report_correlation, ) -from verdi.schemas import WorldEvent +from verdi_agents.schemas import WorldEvent -#: Deterministic seed string recorded in sample and report (ADR-012 §B: the +#: Deterministic seed string recorded in sample and report (the #: sample is a pure function of the tape, no randomness, seed recorded). SAMPLE_SEED = "g4-judge-all-dispatches/v1" -#: ADR-012 §C: judge max_tokens floor (reasoning models can return empty -#: content below this; L1 lesson). +#: Judge max_tokens floor (reasoning models can return empty +#: content below this). MIN_JUDGE_MAX_TOKENS = 1024 #: Retry backoff (seconds) for one outer-loop retry of a failed judge row. RETRY_BACKOFF_SECONDS = 1.0 -#: A1-2 m2: the report's ``agreement`` metric, stated in every report. +#: The report's ``agreement`` metric, stated in every report. AGREEMENT_METRIC_DEFINITION = ( "agreement = (judge_stance == engine_stance) AND code_judgment.correct, " "where engine_stance = yes iff the deterministic engine accepted the " @@ -100,7 +100,7 @@ def wilson_ci_95( ) -> tuple[float, float] | None: """Wilson score 95% confidence interval for a binomial rate. - Stdlib-only (A1-2 M2 program rule: no scipy/numpy). Returns + Stdlib-only (no scipy/numpy). Returns ``(low, high)`` clipped to [0, 1], or ``None`` when ``total == 0`` (no defined rate). Validated in tests against known values (e.g. n=100, x=10 -> ~[0.0552, 0.1744]). @@ -233,13 +233,13 @@ def build_judge_sample( tape_path: Path | str, ledger_path: Path | str, ) -> list[dict[str, Any]]: - """The frozen all-dispatches judge sample (ADR-012 §B), sealed correlated. + """The frozen all-dispatches judge sample, correlated by sealed event id. Sample = ALL dispatch records of the tape, sorted by ``(tick, actor_id, task_id)`` (``seq`` tie-break for total order). It is a pure function of the tape: no randomness, no post-hoc selection after seeing judge output. Every engine-rejection row is therefore in - the sample (event 42 for the L1 run); the guarantee is asserted from + the sample (event 42 in the recorded live run); the guarantee is asserted from the ledger, not hardcoded. """ tape = _load_jsonl(Path(tape_path)) @@ -288,7 +288,7 @@ def build_judge_sample( # --------------------------------------------------------------------------- -# Sealed target: replays the captured L1 assistant response, zero calls +# Sealed target: replays the captured assistant response, zero calls # --------------------------------------------------------------------------- @@ -440,7 +440,7 @@ def call_count(self) -> int: """Number of actual judge-model calls (budget-relevant). Budget-breach rows (calls never made) are audited in the tape but - do not consume the ADR-012 §C call budget. + do not consume the call budget. """ return sum( 1 @@ -461,7 +461,7 @@ def build_judge_rows( ) -> list[dict[str, Any]]: """Per-sample-row code-level judgment + the sealed-dispatch judge prompt. - Reuses ``live_bridge.build_judge_evidence`` / SimpleAudit + Reuses ``trace_audit.build_judge_evidence`` / SimpleAudit ``grade_choice_match`` for the deterministic code judgment (no model calls) and renders the judge prompt (the sealed dispatch: captured proposal verbatim + engine outcome + violation codes). @@ -571,7 +571,7 @@ def _build_auditor( capture: JudgeTapeCapture, replay: str, ) -> ModelAuditor: - """ModelAuditor via its public API (ADR-012 §A) with the sealed target. + """ModelAuditor via its public API with the sealed target. ``temperature``/``max_tokens`` are PER-CALL ``acompletion`` kwargs, so they are applied through ``judge_params`` at run time (see ``_one_scenario``), @@ -621,7 +621,7 @@ def run_live_judge( judge_base_url: str | None = None, judge_api_key: str | None = None, ) -> dict[str, Any]: - """Run the ADR-012 live judge pass and write the judge tape + meta. + """Run the live judge pass and write the judge tape + meta. Returns the raw judge-run record (verdicts + judge tape rows + meta). The DETERMINISTIC report is produced separately by @@ -630,7 +630,7 @@ def run_live_judge( endpoint nondeterminism. - Sample: all dispatches of the tape (``build_judge_sample``); the sole - L1 engine DISPATCH rejection (event 42) is in-sample by construction. + engine DISPATCH rejection of the recorded run (event 42) is in-sample by construction. - Budget: <= ``max_calls`` (default 96) judge calls total (retries count), concurrency 1 (``max_workers=1``); one outer retry per transiently-failed row. @@ -639,7 +639,7 @@ def run_live_judge( ledger, or any scoring input. """ if max_tokens < MIN_JUDGE_MAX_TOKENS: - raise ValueError(f"judge max_tokens must be >= {MIN_JUDGE_MAX_TOKENS} (ADR-012 §C)") + raise ValueError(f"judge max_tokens must be >= {MIN_JUDGE_MAX_TOKENS}") if judge_base_url is None: judge_base_url = base_url if judge_api_key is None: @@ -759,7 +759,7 @@ def _budget_breach(row: dict[str, Any], note: str) -> None: def _attempt(row: dict[str, Any]) -> tuple[str | None, Exception | None]: """One live judge attempt; returns (content, error). - A1-2 Eng M1: catches ANY per-attempt exception (not just + Catches ANY per-attempt exception (not just RuntimeError) so a single bad row is recorded as an audited judge_error row and the run continues; the outer loop only re-raises for a FATAL failure that prevents the tape itself @@ -780,8 +780,8 @@ def _real_calls() -> list[dict[str, Any]]: ] def _write_tape_and_meta(partial: bool, fatal_error: str | None) -> dict[str, Any]: - """Write judge tape + meta. On ``partial=True`` this is the A1-2 - Eng M1 fatal-failure path: rows so far are persisted (never lost) + """Write judge tape + meta. On ``partial=True`` this is the + fatal-failure path: rows so far are persisted (never lost) and the meta is flagged ``partial`` before the caller re-raises.""" tape_text = ( "\n".join(json.dumps(r, sort_keys=True) for r in capture.rows) + "\n" @@ -835,7 +835,7 @@ def _write_tape_and_meta(partial: bool, fatal_error: str | None) -> dict[str, An _write_json(out_dir / "judge_run_meta.json", meta) return meta - # A1-2 Eng M1: the per-row loop may hit a fatal, unexpected error + # The per-row loop may hit a fatal, unexpected error # (e.g. a missing tape field, a SimpleAudit regression). The rows # judged SO FAR must never be lost: on any unexpected exception write # a PARTIAL tape + meta (flagged) and re-raise. Per-attempt judge @@ -912,7 +912,7 @@ async def _sleep_backoff() -> None: def _structural_artifact_note(code_stance: str, engine_stance: str) -> str: - """A1-2 M1: honest wording for a code-judge divergence that is + """Honest wording for a code-judge divergence that is STRUCTURAL, not a reading of the evidence. The code judge (SimpleAudit ``grade_choice_match`` over @@ -954,7 +954,7 @@ def _disagreement_reason(row: dict[str, Any], verdict: dict[str, Any]) -> str: f"Judge reason: {reason}" ) # The model-judge diverges from the engine. Derive the wording from - # WHICH component it aligns with (A1-2 M1: no hardcoded events). + # WHICH component it aligns with (no hardcoded events). if code_stance == jv: return ( f"model-judge ({jv}) and code-judge ({code_stance}) both " @@ -986,7 +986,7 @@ def build_judge_report( meta: dict[str, Any], correlation: dict[str, Any], ) -> dict[str, Any]: - """DETERMINISTIC G4 judge report — a PURE function. + """DETERMINISTIC judge report — a PURE function. Inputs are plain data only: the preregistered sample, the code-level judgment rows, the model-judge verdict rows (from the judge tape), the @@ -1050,9 +1050,9 @@ def build_judge_report( rejected_rows = [r for r in judged if r["engine_decision"] == "rejected"] def _breakdown(sub: list[dict[str, Any]]) -> dict[str, Any]: - """A1-2 M2: per-class agreement with counts + Wilson 95% CI. + """Per-class agreement with counts + Wilson 95% CI. - Classes with a single judged row (e.g. the L1 rejection class, + Classes with a single judged row (e.g. the one rejection class, n=1) are explicitly flagged: a 1/1 rate carries a CI spanning (0, 1) and is NOT evidence about the class. """ diff --git a/src/verdi/audit/simpleaudit_bridge.py b/src/verdi_agents/audit/simpleaudit_bridge.py similarity index 96% rename from src/verdi/audit/simpleaudit_bridge.py rename to src/verdi_agents/audit/simpleaudit_bridge.py index cce196a..51dc060 100644 --- a/src/verdi/audit/simpleaudit_bridge.py +++ b/src/verdi_agents/audit/simpleaudit_bridge.py @@ -4,7 +4,7 @@ declared as a dependency. This module imports ``simpleaudit.tracing`` only when :func:`build_simpleaudit_trace_source` is called and wraps a SimpleAudit ``TraceProvider`` (e.g. ``BuiltinOTLP``) behind the adapter's -structural :class:`~verdi.audit.adapter.TraceSource` protocol. +structural :class:`~verdi_agents.audit.adapter.TraceSource` protocol. If the package is unavailable, :class:`SimpleAuditNotInstalledError` is raised with an actionable message — tests exercise exactly that path. @@ -25,7 +25,7 @@ def __init__(self, detail: str = "") -> None: "PYTHONPATH=/path/to/SimpleAudit:$PYTHONPATH, or set " "VERDI_SIMPLEAUDIT_PATH to the directory containing the " "'simpleaudit' package. If real tracing is not required, use " - "verdi.audit.in_memory.InMemoryTraceSource instead." + "verdi_agents.audit.in_memory.InMemoryTraceSource instead." ) if detail: message += f" (import failure: {detail})" diff --git a/src/verdi/audit/live_bridge.py b/src/verdi_agents/audit/trace_audit.py similarity index 97% rename from src/verdi/audit/live_bridge.py rename to src/verdi_agents/audit/trace_audit.py index 2828292..f890285 100644 --- a/src/verdi/audit/live_bridge.py +++ b/src/verdi_agents/audit/trace_audit.py @@ -1,12 +1,12 @@ -"""L2 G4 live audit over a captured L1 live run (ADR-011). +"""Audit a captured live run: match each model call to its ledger event and grade it. -Runs the existing :class:`~verdi.audit.adapter.AuditAdapter` over the +Runs the existing :class:`~verdi_agents.audit.adapter.AuditAdapter` over the already-captured live tape (``live_tape.jsonl``) with **sealed-event_id correlation** (never positional): each tape record is mapped to the ledger event id of the dispatch outcome (``action_accepted`` / ``action_rejected``) for the same ``(actor_id, task_id)``, deterministically derived from the sealed ledger. An ambiguous tape record is an error — there is no silent -fallback (LESSONS.md correlation rule). +fallback. Where SimpleAudit is reused and where it is deliberately not forced: @@ -20,7 +20,7 @@ they require a live :class:`~simpleaudit.targets.Target` (an HTTP endpoint, a judge LLM client, regenerated transcripts). Re-running them over the captured tape would burn budget and violate the "no new model - calls" contract for this tranche. This module documents that choice per + calls" rule for this audit. This module documents that choice per the "use SimpleAudit where it fits; never force it" rule. The judge remains an **advisor**: every row of the report also carries the @@ -46,10 +46,10 @@ from pathlib import Path from typing import Any -from verdi.audit.adapter import AuditAdapter, TraceSource -from verdi.measurement.scoring import RubricScorer -from verdi.schemas import WorldEvent -from verdi.schemas.probes import AuditRecord, ProbeItem +from verdi_agents.audit.adapter import AuditAdapter, TraceSource +from verdi_agents.measurement.scoring import RubricScorer +from verdi_agents.schemas import WorldEvent +from verdi_agents.schemas.probes import AuditRecord, ProbeItem RUBRIC_ID = "g4-live-dispatch" RUBRIC_VERSION = "1.0" @@ -334,7 +334,7 @@ def sample_disclosure( mapping: dict[int, list[dict[str, Any]]], sample_seqs: list[int], ) -> dict[str, Any]: - """Compute the honest sample disclosure (independent-review fix, F1/F2). + """Compute the honest sample disclosure. Every value is derived from the data, not hardcoded: judged/unjudged counts from ``len(sample)``/``len(tape)``, the in-sample accepted/ @@ -374,7 +374,7 @@ def sample_disclosure( def sample_disclosure_limitations(d: dict[str, Any]) -> list[str]: - """Limitation strings computed from :func:`sample_disclosure` (F1/F2).""" + """Limitation strings computed from :func:`sample_disclosure`.""" lines: list[str] = [] lines.append( f"Judge coverage: {d['judged']} judged / {d['unjudged']} unjudged of " @@ -526,7 +526,7 @@ def audit_live_run( sample_size: int = DEFAULT_SAMPLE_SIZE, scorer_factory: Callable[..., RubricScorer] = RubricScorer, ) -> dict[str, Any]: - """Run the G4 audit over a captured live run and write a deterministic report. + """Run the trace audit over a captured live run and write a deterministic report. - Correlation: each sealed dispatch event <-> exactly one span, by sealed event id (never positional). Every sealed dispatch event must diff --git a/src/verdi/cli.py b/src/verdi_agents/cli.py similarity index 54% rename from src/verdi/cli.py rename to src/verdi_agents/cli.py index 0f2d5a1..b422eda 100644 --- a/src/verdi/cli.py +++ b/src/verdi_agents/cli.py @@ -25,12 +25,12 @@ def init(target: Path, *, force: bool = False) -> list[Path]: src = examples_dir() files = sorted(p for p in src.rglob("*.yaml") if p.is_file()) if not files: - raise SystemExit(f"verdi init: no examples found in {src}") + raise SystemExit(f"verdi-agents init: no examples found in {src}") dest_root = target / "examples" clashes = [dest_root / p.relative_to(src) for p in files if (dest_root / p.relative_to(src)).exists()] if clashes and not force: - raise SystemExit("verdi init: these files already exist (use --force to overwrite):\n " + raise SystemExit("verdi-agents init: these files already exist (use --force to overwrite):\n " + "\n ".join(str(c) for c in clashes)) written = [] for path in files: @@ -47,7 +47,7 @@ def init(target: Path, *, force: bool = False) -> list[Path]: def _init_main(argv: list[str]) -> int: parser = argparse.ArgumentParser( - prog="verdi init", + prog="verdi-agents init", description="Create a folder with ready-to-run example scenarios and sweeps.") parser.add_argument("directory", nargs="?", default=".", help="where to create it (default: here)") parser.add_argument("--force", action="store_true", help="overwrite existing example files") @@ -62,15 +62,15 @@ def _init_main(argv: list[str]) -> int: Next: {where}# 1. put your endpoint key in .env (MODEL_API_KEY=...) # 2. set base_url/name in examples/starter.yaml's `model:` block - verdi run examples/starter.yaml # results: runs/starter/report.html - verdi sweep examples/sweeps/model_comparison.yaml --dry-run + verdi-agents run examples/starter.yaml # results: runs/starter/report.html + verdi-agents sweep examples/sweeps/model_comparison.yaml --dry-run (Not installed? Use uvx: uvx verdi-agents run ...)""") return 0 def _run_main(argv: list[str]) -> int: - """`verdi run `: one simulation; run dir defaults to runs/.""" - from verdi.society.live import main as live_main + """`verdi-agents run `: one simulation; run dir defaults to runs/.""" + from verdi_agents.society.live import main as live_main if argv and not argv[0].startswith("-"): scenario, rest = argv[0], argv[1:] @@ -80,28 +80,57 @@ def _run_main(argv: list[str]) -> int: return live_main(argv) +def _positional_to_flag(argv: list[str], flag: str) -> list[str]: + """Let `cmd FILE ...` mean `cmd --flag FILE ...` (both spellings work).""" + if argv and not argv[0].startswith("-"): + return [flag, argv[0], *argv[1:]] + return argv + + +def _pair_main(argv: list[str]) -> int: + from verdi_agents.experiments.paired_run import main as pair_main + + return pair_main(_positional_to_flag(argv, "--config")) + + +def _compare_main(argv: list[str]) -> int: + from verdi_agents.analysis.outcome_metrics import main as compare_main + + return compare_main(_positional_to_flag(argv, "--pair-root")) + + +def _sweep_main(argv: list[str]) -> int: + from verdi_agents.experiments.sweep import main as sweep_main + + return sweep_main(argv) + + +COMMANDS = { + "init": (_init_main, "create a folder with example scenarios and sweeps"), + "run": (_run_main, "run one society (verdi-agents run examples/starter.yaml)"), + "sweep": (_sweep_main, "rerun a scenario many times, changing only what you list"), + "pair": (_pair_main, "run the same society twice: fixed rules vs rules agents can change"), + "compare": (_compare_main, "measure and compare the two halves of a `pair` run"), +} + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) - if argv[:1] == ["sweep"]: - from verdi.experiments.sweep import main as sweep_main - - return sweep_main(argv[1:]) - if argv[:1] == ["init"]: - return _init_main(argv[1:]) - if argv[:1] == ["run"]: - return _run_main(argv[1:]) - parser = argparse.ArgumentParser(prog="verdi", description="Verdi research testbed CLI", - epilog="commands:\n" - " init create a folder with example scenarios " - "and sweeps\n" - " run run one scenario " - "(verdi run examples/starter.yaml)\n" - " sweep rerun a scenario many times, changing " - "only what you list (verdi sweep --help)", - formatter_class=argparse.RawDescriptionHelpFormatter) + if argv and argv[0] in COMMANDS: + handler, _ = COMMANDS[argv[0]] + return handler(argv[1:]) + width = max(len(name) for name in COMMANDS) + epilog = "commands:\n" + "\n".join( + f" {name.ljust(width)} {help_text}" for name, (_, help_text) in COMMANDS.items() + ) + "\n\nRun `verdi-agents --help` for the options of one command." + parser = argparse.ArgumentParser( + prog="verdi-agents", + description="Simulate small societies of AI agents and watch how their values and " + "rules change over time.", + epilog=epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.parse_args(argv) - # No subcommand given: show usage (the old text pointed at a doc that - # does not exist). parser.print_help() return 0 diff --git a/src/verdi/config.py b/src/verdi_agents/config.py similarity index 100% rename from src/verdi/config.py rename to src/verdi_agents/config.py diff --git a/src/verdi/experiments/__init__.py b/src/verdi_agents/experiments/__init__.py similarity index 60% rename from src/verdi/experiments/__init__.py rename to src/verdi_agents/experiments/__init__.py index df4b7bc..e56b70d 100644 --- a/src/verdi/experiments/__init__.py +++ b/src/verdi_agents/experiments/__init__.py @@ -1,16 +1,16 @@ -"""P2 controlled recovery experiments (WS-2 public surface). +"""Controlled recovery experiments: regimes, treatments and matched pairs. -Re-exports the public names other workstreams may import: regime +Re-exports the public names other modules may import: regime configuration, record-only deterministic treatments and matched pairs. """ -from verdi.experiments.pairs import MatchedPair, make_matched_pair, tape_hash -from verdi.experiments.regimes import ( +from verdi_agents.experiments.pairs import MatchedPair, make_matched_pair, tape_hash +from verdi_agents.experiments.regimes import ( REGIME_LABELS, RegimeConfig, load_regime, manifest_metadata, ) -from verdi.experiments.treatments import ( +from verdi_agents.experiments.treatments import ( TreatmentOutcome, placebo_treatment, rights_restoration_treatment, diff --git a/src/verdi/experiments/g5_pairs.py b/src/verdi_agents/experiments/counterfactual_pairs.py similarity index 84% rename from src/verdi/experiments/g5_pairs.py rename to src/verdi_agents/experiments/counterfactual_pairs.py index 913ec37..6c673a0 100644 --- a/src/verdi/experiments/g5_pairs.py +++ b/src/verdi_agents/experiments/counterfactual_pairs.py @@ -1,51 +1,50 @@ -"""ADR-013 G5 live paired arms: matched treatment/placebo counterfactuals. - -FROZEN design — ``docs/ADR-013-G5-PAIRED-ARMS-PROPOSAL.md`` (this file implements -that contract; ADR-013 is authoritative over any paraphrase): - -* ONE L1 live run is the sealed origin. Its ledger prefix (events 0..49, - through ``appeal_filed``) and its captured live tape (81 verbatim model - calls) are byte-sealed. Branch point: AFTER sealed event 49. -* Per pair ``p`` in 0..9, ONE shared suffix tape ``R_p`` — the post-branch - pilot segment (lifecycle ticks 13..29: routine dispatches + tick-17 - assembly + post-tick probes) — is generated LIVE with fresh Qwen calls - (concurrency 1, hard cap 700 calls) by driving the pilot once with the - sealed prefix REPLAYED from the L1 tape and the remaining segment answered - by the live endpoint. R_p is captured verbatim in the ``live_tape.jsonl`` - format (the pair tape = 34 replayed prefix rows + 47 live suffix rows). +"""Run matched treatment/placebo pairs of one society to measure a recovery intervention. + +How it works: + +* One earlier live run is the fixed origin. Its ledger prefix (events 0..49, + through ``appeal_filed``) and its recorded live tape (81 verbatim model + calls) are sealed. The branch point is AFTER sealed event 49. +* For each pair ``p`` in 0..9 there is ONE shared suffix tape ``R_p``: the + rest of the reference run after the branch (lifecycle ticks 13..29: routine + dispatches + the tick-17 assembly + post-tick probes). It is generated LIVE + with fresh model calls (concurrency 1, hard cap 700 calls) by driving the + reference run once, replaying the sealed prefix from the origin tape and + sending the remaining calls to the live endpoint. R_p is saved verbatim in + the ``live_tape.jsonl`` format (34 replayed prefix rows + 47 live rows). * R_p is REPLAYED BYTE-IDENTICAL for both arms of pair ``p`` (TapeReplayCaller): * treatment arm T_p: prefix + ``appeal_upheld`` + ``rights_restored`` steps (deterministic scripted governance, ``adjudicate_appeal(True, ...)``); - * placebo arm P_p: prefix + the VERBATIM L1 ``appeal_dismissed`` event 50 + * placebo arm P_p: prefix + the VERBATIM origin ``appeal_dismissed`` event 50 (the driver's default scripted adjudication) + R_p. Only the adjudication governance event(s) at the branch position differ. -* Matching (experiments/pairs.py): both arms share the sealed origin snapshot - hash, the sealed prefix tape hash (same hash chain), and the identical R_p. - Matched counterfactual pairs — NOT randomized. No causal wording. -* Scaffold invariance: pilot_v1 action kinds are non-mutating, so both arms - converge to an IDENTICAL final world-state hash; model variance lives in the - ledger chain (event hashes), not in state. - -MANDATORY placebo-inert null control (ADR-013 Addendum A1, offline, BEFORE -any live call): fold L1's own recorded tape as BOTH arms under the CURRENT -code and assert the same-code invariants — sealed prefix 0..49 byte-identical; -the two arm ledgers byte-identical EXCEPT at the adjudication events (and the -chain-bookkeeping fields that cascade from the branch); both arms' folded -final state hashes equal (scaffold invariance); both hash chains unbroken. -Divergence is a pipeline bug and BLOCKS the live run. The retired v1 -standard ("folded suffix byte-identical to historical L1 ledger bytes") is -NOT asserted: L1 was recorded at a different code version (see -``L1_GIT_SHA``) and the live placebo arms carry fresh model outputs, so -byte-identity to historical L1 is impossible and was a false positive. - -Replay provenance (L1, READ ONLY): ``/Users/sushantgautam/Documents/verdi/runs/live/20261010T071750Z/``. -Credentials come exclusively from environment variables; no key ever appears -in any artifact (audited by :func:`audit_artifacts`). +* Matching (experiments/pairs.py): both arms share the origin snapshot hash, + the sealed prefix tape hash (same hash chain), and the identical R_p. + These are matched counterfactual pairs, NOT randomized; no causal wording. +* Scaffold invariance: the ``pilot_v1`` scenario's action kinds do not change + world state, so both arms reach an IDENTICAL final world-state hash; model + variance shows up in the ledger chain (event hashes), not in state. + +Required placebo-inert null control (offline, BEFORE any live call): replay +the origin run's own recorded tape as BOTH arms under the CURRENT code and +check that the sealed prefix 0..49 is byte-identical; the two arm ledgers are +byte-identical EXCEPT at the adjudication events (and the chain-bookkeeping +fields that follow from the branch); both arms' final state hashes are equal; +both hash chains are unbroken. Any difference is a pipeline bug and BLOCKS the +live run. An older check ("replayed suffix byte-identical to the historical +origin ledger bytes") is NOT made: the origin run was recorded at a different +code version (see ``L1_GIT_SHA``) and the live placebo arms carry fresh model +outputs, so that byte-identity cannot hold. + +Origin run directory (read only): see ``L1_RUN_DIR``. +Credentials come only from environment variables; no key ever appears in any +artifact (checked by :func:`audit_artifacts`). """ from __future__ import annotations import json +import os import re import shutil import statistics @@ -54,41 +53,46 @@ from pathlib import Path from typing import Any -from verdi.agents.client import ModelError, ModelResult -from verdi.analysis.estimates import ( +from verdi_agents.agents.client import ModelError, ModelResult +from verdi_agents.analysis.estimates import ( ESTIMANDS, paired_estimates, run_estimand_values, ) -from verdi.analysis.report import ( +from verdi_agents.analysis.report import ( _validate_arm_run_dict, build_g5_report, write_g5_report, ) -from verdi.config import ModelEndpointConfig, model_endpoint_from_env -from verdi.pilot.driver import PilotDriver, PilotRun -from verdi.pilot.live import LiveModelAdapter, TapeReplayCaller, _ClientFactory -from verdi.schemas.actions import ProposedAction, ValidationResult -from verdi.schemas.events import WorldEvent -from verdi.world.log import EventLog, LogIntegrityError -from verdi.world.reducer import WorldReducer +from verdi_agents.config import ModelEndpointConfig, model_endpoint_from_env +from verdi_agents.reference_run.driver import PilotDriver, PilotRun +from verdi_agents.reference_run.live import LiveModelAdapter, TapeReplayCaller, _ClientFactory +from verdi_agents.schemas.actions import ProposedAction, ValidationResult +from verdi_agents.schemas.events import WorldEvent +from verdi_agents.world.log import EventLog, LogIntegrityError +from verdi_agents.world.reducer import WorldReducer class G5PipelineError(RuntimeError): - """An ADR-013 §5 STOP condition; the live run must halt with evidence.""" + """A stop condition: the live run must halt and report its evidence.""" # --------------------------------------------------------------------------- # -# Frozen constants (ADR-013) +# Fixed design constants # --------------------------------------------------------------------------- # -L1_RUN_DIR = Path("/Users/sushantgautam/Documents/verdi/runs/live/20261010T071750Z") +#: Default origin run: the recorded live run inside a source checkout. Point +#: ``VERDI_ORIGIN_RUN_DIR`` (or the ``run_dir`` argument) at your own copy. +L1_RUN_DIR = Path( + os.environ.get("VERDI_ORIGIN_RUN_DIR") + or Path(__file__).resolve().parents[3] / "runs" / "live" / "20261010T071750Z" +) L1_TAPE_PATH = L1_RUN_DIR / "live_tape.jsonl" L1_EVENT_LOG_PATH = L1_RUN_DIR / "event_log.jsonl" #: Branch point: AFTER sealed event 49 (appeal_filed). Prefix = 0..49. BRANCH_EVENT_ID = 49 -#: L1 event 50 = verbatim appeal_dismissed = the placebo step. +#: Origin event 50 = verbatim appeal_dismissed = the placebo step. PLACEBO_EVENT_ID = 50 #: R_p spans lifecycle ticks 13..29. SUFFIX_TICK_MIN = 13 @@ -101,15 +105,15 @@ class G5PipelineError(RuntimeError): N_PAIRS = 10 HARD_CALL_CAP = 700 EXPECTED_SUFFIX_CALLS = 47 -#: ADR-013 §5: bounded cross-pair retries on transient endpoint errors +#: Bounded cross-pair retries on transient endpoint errors #: (on top of ModelClient's own per-request retry budget). _TRANSIENT_RETRY_LIMIT = 3 #: Qwen3.8-27B is a reasoning model; propose() passes max_tokens=None and the #: completion can come back empty when the budget runs out inside reasoning. #: Force a floor on LIVE calls only (never replay), so taped rows stay -#: byte-identical to L1 format. L1's own completions reached 6307 tokens -#: (reasoning-dominated); pair_0 at 2048 truncated 12/47 calls to -#: text_tokens=0, so the floor is 8192 (headroom over L1's observed max). +#: byte-identical to the origin tape format. The origin run's completions +#: reached 6307 tokens (mostly reasoning); pair_0 at 2048 truncated 12/47 +#: calls to text_tokens=0, so the floor is 8192 (headroom over that max). LIVE_MAX_TOKENS = 8192 TREATMENT_GROUNDS = "upheld-appeal treatment arm" @@ -120,44 +124,44 @@ class G5PipelineError(RuntimeError): _TREATMENT_ONLY_KINDS = frozenset({"appeal_upheld", "rights_restored"}) _PLACEBO_ONLY_KINDS = frozenset({"appeal_dismissed"}) -#: Version-drift disclosure (ADR-013 Addendum A1): L1 was recorded at this -#: code version; the live G5 run executed at this one. Four driver commits -#: separate them — cross-version byte-identity with historical L1 was never -#: a well-posed standard (see the null-control docstring). +#: Version-drift disclosure: the origin run (L1_GIT_SHA) was recorded at one +#: code version and the live paired run (G5_RUN_GIT_SHA) ran at another. Four +#: driver commits separate them, so byte-identity across the two versions +#: was never a valid check (see the null-control docstring). L1_GIT_SHA = "eb4a2a00c29c83bee3847ab14cdda0147ea8e053" G5_RUN_GIT_SHA = "c455958d82900e52beb3d02b0b8bdd23a434941b" # The frozen report wording contains exactly these three NEGATED "causal" -# occurrences (WS-4 mandated framing, analysis/report.py). No NEW occurrences +# occurrences (required framing, see analysis/report.py). No NEW occurrences # may be introduced by the paired extensions below. _FROZEN_CAUSAL_OCCURRENCES = 3 # --------------------------------------------------------------------------- # -# L1 provenance loading (read-only) +# Loading the origin run (read-only) # --------------------------------------------------------------------------- # @dataclass(frozen=True) class L1Provenance: - """Sealed L1 origin: verified ledger + verbatim tape rows.""" + """Sealed origin run: verified ledger + verbatim tape rows.""" event_log: EventLog tape_rows: list[dict[str, Any]] = field(compare=False) @property def prefix_events(self) -> tuple[WorldEvent, ...]: - """Sealed prefix: L1 events 0..BRANCH_EVENT_ID inclusive (50 events).""" + """Sealed prefix: origin events 0..BRANCH_EVENT_ID inclusive (50 events).""" return self.event_log.events[: BRANCH_EVENT_ID + 1] @property def prefix_tape_rows(self) -> list[dict[str, Any]]: - """L1 tape rows for lifecycle tick <= 12 (replayed, never live).""" + """Origin tape rows for lifecycle tick <= 12 (replayed, never live).""" return [r for r in self.tape_rows if int(r["lifecycle_tick"]) <= 12] @property def suffix_tape_rows(self) -> list[dict[str, Any]]: - """L1's own R (lifecycle ticks 13..29) — used by the null control.""" + """The origin run's own suffix (lifecycle ticks 13..29), used by the null control.""" return [ r for r in self.tape_rows @@ -166,12 +170,12 @@ def suffix_tape_rows(self) -> list[dict[str, Any]]: @property def placebo_event(self) -> WorldEvent: - """Verbatim L1 ``appeal_dismissed`` (event 50).""" + """Verbatim origin ``appeal_dismissed`` (event 50).""" ev = self.event_log.events[PLACEBO_EVENT_ID] if ev.kind != "appeal_dismissed": raise G5PipelineError( - f"L1 event {PLACEBO_EVENT_ID} is {ev.kind!r}, expected " - "appeal_dismissed — L1 provenance moved" + f"origin event {PLACEBO_EVENT_ID} is {ev.kind!r}, expected " + "appeal_dismissed — origin run changed" ) return ev @@ -180,7 +184,7 @@ def origin_state_hash(self) -> str: """Terminal after_hash of the sealed prefix (event 49).""" after = self.prefix_events[-1].after_hash if not after: - raise G5PipelineError("L1 branch event 49 has no after_hash") + raise G5PipelineError("origin branch event 49 has no after_hash") return after @@ -192,10 +196,10 @@ def _count_log_events(path: str | Path) -> int: def load_l1(run_dir: str | Path = L1_RUN_DIR) -> L1Provenance: - """Load and sanity-check the sealed L1 ledger + tape (read-only).""" + """Load and sanity-check the sealed origin ledger + tape (read-only).""" run_dir = Path(run_dir) if not (run_dir / "live_tape.jsonl").is_file(): - raise G5PipelineError(f"L1 tape missing: {run_dir}") + raise G5PipelineError(f"origin tape missing: {run_dir}") tape_rows = [ json.loads(line) for line in (run_dir / "live_tape.jsonl").read_text(encoding="utf-8").splitlines() @@ -205,13 +209,13 @@ def load_l1(run_dir: str | Path = L1_RUN_DIR) -> L1Provenance: n_expected = _count_log_events(L1_EVENT_LOG_PATH) if len(log.events) != n_expected: raise G5PipelineError( - f"L1 ledger has {len(log.events)} events, expected {n_expected} " + f"origin ledger has {len(log.events)} events, expected {n_expected} " "(derived from the sealed event-log file length)" ) if log.events[BRANCH_EVENT_ID].kind != "appeal_filed": - raise G5PipelineError("L1 event 49 is not appeal_filed — provenance moved") + raise G5PipelineError("origin event 49 is not appeal_filed — origin run changed") if l1_prefix_count(tape_rows) != 34: - raise G5PipelineError("L1 tape prefix row count is not 34 — provenance moved") + raise G5PipelineError("origin tape prefix row count is not 34 — origin run changed") return L1Provenance(event_log=log, tape_rows=tape_rows) @@ -240,7 +244,7 @@ class _MaxTokensGuard: """Force a max_tokens floor on ONE downstream caller (reasoning model). Applied ONLY to the live side; replayed rows never pass through it, so - captured prefix rows stay byte-identical to the L1 tape. + captured prefix rows stay byte-identical to the origin tape. """ def __init__(self, inner: LiveModelAdapter, floor: int) -> None: @@ -273,9 +277,9 @@ async def generate_structured( class PrefixReplayLiveCaller: """ModelCaller for ONE R_p generation (live, per pair). - Serves the sealed prefix from the L1 tape (ticks <= 12, seq order), then - answers the remaining segment with the LIVE endpoint (one ModelClient per - dispatch). The pilot's live-capture path records every call — including + Serves the sealed prefix from the origin tape (ticks <= 12, seq order), + then answers the remaining segment with the LIVE endpoint (one ModelClient + per dispatch). The reference run's live-capture path records every call — including the replayed prefix rows — into the pair's ``live_tape.jsonl``; R_p is the tick >= 13 rows of that file. The run is fully deterministic in its scaffolding; only the suffix responses are fresh model outputs. @@ -341,7 +345,7 @@ def generate_pair_tape( ``EXPECTED_SUFFIX_CALLS`` live endpoint calls; bounded retries on 429/5xx are inside ModelClient. Budget is enforced by the caller via ``used_calls`` and the ``HARD_CALL_CAP``. Persistent endpoint failure - propagates and must be handled per ADR-013 §5 (STOP + evidence). + propagates; the caller must stop and report the evidence. Quarantined (unparseable) proposals are returned, not raised. """ if used_calls + EXPECTED_SUFFIX_CALLS > HARD_CALL_CAP: @@ -375,8 +379,8 @@ def generate_pair_tape( f"live caller made {caller.live_call_count} calls, expected {EXPECTED_SUFFIX_CALLS}" ) # Quarantined (unparseable) proposals are recorded model behavior, NOT a - # pipeline bug: L1 format tolerates them (its own ledger folds rejected - # actions). They are surfaced as audit evidence, not a hard stop. + # pipeline bug: the origin tape format allows them (its own ledger folds + # rejected actions). They are surfaced as audit evidence, not a hard stop. quarantined = [ (r.get("task_id"), r.get("quarantine_reason")) for r in live_rows if r.get("quarantined") ] @@ -404,16 +408,16 @@ def _diffing_fields(a: Any, b: Any) -> set[str]: def null_control(l1: L1Provenance, repo_root: str | Path) -> dict[str, Any]: - """Placebo-inert null control — SAME-CODE invariants (ADR-013 Addendum A1). + """Placebo-inert null control: check SAME-CODE invariants before going live. - Folds L1's own recorded tape (81 verbatim rows) as BOTH arms under the + Folds the origin run's own recorded tape (81 verbatim rows) as BOTH arms under the CURRENT code and asserts, on the two same-code folds: (a) the sealed prefix (events 0..49) is byte-identical in both arm - ledgers and to the sealed L1 ledger; + ledgers and to the sealed origin ledger; (b) the two arm ledgers are byte-identical EXCEPT at the adjudication events (treatment = ``appeal_upheld`` + ``rights_restored``, placebo - = the verbatim L1 ``appeal_dismissed``) and in the chain-bookkeeping + = the verbatim origin ``appeal_dismissed``) and in the chain-bookkeeping fields that cascade from the branch (``event_id``, ``cause_event_id``, ``tick``, ``event_hash``, ``prev_hash``); ``kind``, ``payload`` and ``decision`` must match event-for-event; @@ -421,20 +425,20 @@ def null_control(l1: L1Provenance, repo_root: str | Path) -> dict[str, Any]: invariance); (d) both hash chains verify end-to-end (0 breaks). - RETIRED (v1 standard): "the folded suffix must be byte-identical to - historical L1 ledger bytes". That comparison is structurally impossible - across code versions (L1 was recorded at ``L1_GIT_SHA``, this fold runs - at ``G5_RUN_GIT_SHA``) and impossible with fresh live model outputs in - the live placebo arms; asserting it produced a false positive. The - returned evidence therefore carries NO byte-identity claim against L1 - and cannot. Any invariant failure raises :class:`G5PipelineError` + Dropped older check: "the folded suffix must be byte-identical to the + historical origin ledger bytes". That comparison cannot hold across code + versions (the origin was recorded at ``L1_GIT_SHA``, this fold runs at + ``G5_RUN_GIT_SHA``) nor with fresh live model outputs in the placebo + arms; asserting it gave a false positive. The returned evidence therefore + makes NO byte-identity claim against the origin run. Any invariant + failure raises :class:`G5PipelineError` (DIVERGENCE): the live run is BLOCKED until the pipeline bug is fixed. """ t_run = run_arm(repo_root, l1.tape_rows, TREATMENT_ARM) p_run = run_arm(repo_root, l1.tape_rows, PLACEBO_ARM) failures: list[str] = [] - # (a) sealed prefix 0..49 byte-identical: T fold == P fold == L1. + # (a) sealed prefix 0..49 byte-identical: T fold == P fold == origin. prefix_ok = True for i in range(0, BRANCH_EVENT_ID + 1): t_dump = t_run.log.events[i].model_dump(mode="json") @@ -553,21 +557,22 @@ def null_control(l1: L1Provenance, repo_root: str | Path) -> dict[str, Any]: def matching_context(l1: L1Provenance, repo_root: str | Path | None = None) -> dict[str, Any]: """Sealed origin snapshot hash + sealed prefix tape hash (all arms/pairs). - B3(a)(b)(c) satisfied structurally: origin hash == L1 event 49 + The origin checks hold by construction: origin hash == origin event 49 ``after_hash`` == canonical_state_hash of the folded prefix terminal state; prefix count == 50 == len(prefix ledger). """ - from verdi.experiments.pairs import tape_hash + from verdi_agents.experiments.pairs import tape_hash origin = l1.origin_state_hash prefix = prefix_event_log(l1) - # B3 cross-check: fold the prefix through a fresh reducer to confirm the + # Cross-check: fold the prefix through a fresh reducer to confirm the # terminal canonical state hash equals the sealed after_hash. state = _fold_prefix_state(l1, repo_root) folded_hash = WorldReducer.canonical_state_hash(state) if folded_hash != origin: raise G5PipelineError( - f"B3 violation: folded prefix terminal hash {folded_hash} != sealed after_hash {origin}" + f"origin hash mismatch: folded prefix terminal hash {folded_hash} " + f"!= sealed after_hash {origin}" ) return { "branch_event_id": BRANCH_EVENT_ID, @@ -580,7 +585,7 @@ def matching_context(l1: L1Provenance, repo_root: str | Path | None = None) -> d def _fold_prefix_state(l1: L1Provenance, repo_root: str | Path | None = None): - """Fold L1 events 0..49 through a fresh WorldReducer (no side effects). + """Fold origin events 0..49 through a fresh WorldReducer (no side effects). Rebuilds the exact reducer transitions the driver performs: action events carry their recorded arguments; governance events use @@ -633,17 +638,17 @@ def _is_governance_kind(kind: str) -> bool: def _t0_state(repo_root: str | Path | None = None): - """The pilot_v1 t0 WorldState from the scenario loader (pure read). + """The ``pilot_v1`` scenario's t0 WorldState from the scenario loader (pure read). Resolves the repo root exactly as :class:`PilotDriver` does (explicit arg, else package-relative parents[3]). """ - from verdi.analysis.scenario import ScenarioLoader + from verdi_agents.analysis.scenario import ScenarioLoader root = Path(repo_root) if repo_root else Path(__file__).resolve().parents[3] loader = ScenarioLoader( - root / "experiments" / "scenarios" / "pilot_v1.json", - root / "experiments" / "task_tape" / "pilot_v1.json", + root / "tests" / "fixtures" / "reference_run" / "scenarios" / "pilot_v1.json", + root / "tests" / "fixtures" / "reference_run" / "task_tape" / "pilot_v1.json", ) return loader.load().t0_state @@ -685,7 +690,7 @@ def matched_pair( context: Mapping[str, Any], ): """Validate the pair identity via experiments/pairs.make_matched_pair.""" - from verdi.experiments.pairs import make_matched_pair + from verdi_agents.experiments.pairs import make_matched_pair return make_matched_pair( arm_manifest(l1, pair, TREATMENT_ARM, context), @@ -709,7 +714,7 @@ def run_arm( Treatment arm: ``appeal_adjudication=(True, TREATMENT_GROUNDS)`` → ``appeal_upheld`` + ``rights_restored`` at tick 12. Placebo arm: ``appeal_adjudication=None`` → the driver's default scripted dismissal, - byte-identical to L1 event 50. Live capture OFF (replay only). + byte-identical to origin event 50. Live capture OFF (replay only). """ if arm == TREATMENT_ARM: adjudication: tuple[bool, str] | None = (True, TREATMENT_GROUNDS) @@ -733,7 +738,7 @@ def arm_run_dict(run: PilotRun, arm: str) -> dict[str, Any]: ``state_snapshots`` keeps real :class:`WorldState` objects (in memory) so ``build_g5_report``'s trajectory hashing works; ``estimates`` carries the - six PREREG-registered estimand values computed over the post-branch + six pre-registered estimand values computed over the post-branch snapshots (ticks 17/24/29) plus the full ledger. """ applied = arm == TREATMENT_ARM @@ -772,7 +777,7 @@ def arm_run_dict(run: PilotRun, arm: str) -> dict[str, Any]: def audit_artifacts(run_root: str | Path) -> dict[str, Any]: - """Mandatory audits over a g5_pairs run directory (ADR-013 §2). + """Required audits over a counterfactual-pairs run directory. (a) no API-key value in ANY artifact text; (b) no ``api_key`` field in any meta; (c) R_p arm-blindness: R_p (tick >= 13) prompts/tape contain no @@ -781,7 +786,7 @@ def audit_artifacts(run_root: str | Path) -> dict[str, Any]: any finding. """ run_root = Path(run_root) - from verdi.config import api_key_values + from verdi_agents.config import api_key_values keys = [k.encode() for k in api_key_values()] findings: list[str] = [] @@ -833,7 +838,7 @@ def audit_artifacts(run_root: str | Path) -> dict[str, Any]: if kind in banned: findings.append(f"cross-arm leak: {kind!r} present in {arm} ledger {log_path}") if findings: - raise G5PipelineError("G5 artifact audit FAILURES: " + "; ".join(findings[:5])) + raise G5PipelineError("paired-run artifact audit FAILURES: " + "; ".join(findings[:5])) return { "scanned_root": str(run_root), "api_key_value_scanned": bool(keys), @@ -844,17 +849,17 @@ def audit_artifacts(run_root: str | Path) -> dict[str, Any]: # --------------------------------------------------------------------------- # -# Deterministic G5 paired report +# Deterministic paired report # --------------------------------------------------------------------------- # -#: Paired-report limitations (ADR-013 Addendum A1). The two frozen entries +#: Paired-report limitations. The two earlier fixed entries #: "recorded-output pilot: no live LLM interaction…" and "single-pair CIs #: withheld: n_pairs < 10…" are STALE for this report — this is a live #: 10-pair run that emits pair CIs — and are replaced below. The #: "determinism is limited to the recorded tape" and "no causal claims" #: entries are carried over VERBATIM (the latter holds the 2 of the 3 -#: frozen negated "causal" occurrences the guard below enforces). +#: fixed negated "causal" occurrences the guard below enforces). _LIMITATIONS_LIVE_10PAIR = ( "live 10-pair run: each pair's remaining tape R_p was generated with " "fresh Qwen3.8-27B calls (470 calls, 0 quarantines); replay applies " @@ -884,7 +889,7 @@ def _cross_pair_spread( ) -> dict[str, Any]: """Numeric per-arm, per-estimand population std across all pairs. - ADR-013 §1: between-pair model variance is reported as a secondary + Between-pair model variance is reported as a secondary descriptive statistic, labeled replication spread — NOT sampling noise. """ spread: dict[str, Any] = {} @@ -906,17 +911,17 @@ def build_g5_paired_report( context: Mapping[str, Any], pairs_run_dicts: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]] = (), ) -> dict[str, Any]: - """Byte-stable G5 report over all 10 pairs (pure over the inputs). + """Byte-stable paired report over all 10 pairs (pure over the inputs). - Extends the FROZEN ``build_g5_report`` (pair-0 arms as the trajectory + Extends the fixed ``build_g5_report`` (pair-0 arms as the trajectory exemplar) with: the same-code null-control evidence - (``null_control_placebo_inert``, ADR-013 Addendum A1), an accurate - RELABEL of the frozen ``null_control`` block (which reports T−P + (``null_control_placebo_inert``), an accurate + RELABEL of the fixed ``null_control`` block (which reports T−P per-estimand differences — the TREATMENT contrast, not a placebo-inert check), the sealed-origin matching context, the numeric cross-pair - replication spread, Addendum A1 framings (false-punishment direction, + replication spread, the required framings (false-punishment direction, #6 restoration-blindness), and corrected limitations. NO new "causal" - wording beyond the three frozen negated occurrences. + wording beyond the three fixed negated occurrences. """ _validate_arm_run_dict("treatment", treatment_run) _validate_arm_run_dict("placebo", placebo_run) @@ -1019,7 +1024,7 @@ def build_g5_paired_report( class _ReportPilotProxy: - """Minimal ``pilot`` object for build_g5_report: manifest + final state + """Minimal reference-run object for build_g5_report: manifest + final state + state_snapshots from the exemplar arm's serialized trajectory.""" def __init__(self, treatment_run: dict[str, Any], context: Mapping[str, Any]) -> None: @@ -1097,11 +1102,11 @@ def run_g5_pairs( report_path: str | Path, config: ModelEndpointConfig | None = None, ) -> dict[str, Any]: - """Full ADR-013 live loop: null control → 10 pairs → estimands → report. + """Full live loop: null control → 10 pairs → estimands → report. Concurrency 1, hard cap 700, Qwen3.8-27B only. Script-style: not a test, no network in CI. Persistent endpoint failure → G5PipelineError with - evidence; the caller must STOP and report BLOCKED per ADR-013 §5. + evidence; the caller must stop and report the run as blocked. """ import subprocess @@ -1125,7 +1130,7 @@ def run_g5_pairs( def _generate_with_transient_retry( pair_dir: Path, ) -> tuple[list[dict[str, Any]], list[tuple[str, Any]]]: - """ADR-013 §5: on transient endpoint errors (429/5xx), back off and + """On transient endpoint errors (429/5xx), back off and retry within a bounded budget; persistent failure raises G5PipelineError (caller must STOP and report BLOCKED). @@ -1153,7 +1158,7 @@ def _generate_with_transient_retry( if not transient: raise G5PipelineError( f"non-transient endpoint failure (HTTP " - f"{exc.status_code}): {exc!r} — STOP per ADR-013 §5" + f"{exc.status_code}): {exc!r} — stopping the run" ) from exc last = exc if attempt >= _TRANSIENT_RETRY_LIMIT: @@ -1182,8 +1187,8 @@ def _generate_with_transient_retry( shutil.rmtree(pair_dir, ignore_errors=True) raise G5PipelineError( f"persistent endpoint failure on pair {pair} after " - f"{_TRANSIENT_RETRY_LIMIT} attempts: {last!r} — STOP per " - f"ADR-013 §5, report BLOCKED with this evidence" + f"{_TRANSIENT_RETRY_LIMIT} attempts: {last!r} — stopping; " + f"report the run as blocked with this evidence" ) for pair in range(N_PAIRS): @@ -1248,7 +1253,7 @@ def _generate_with_transient_retry( json.dumps(_serializable_run_dict(p_dict), indent=2, sort_keys=True) + "\n", encoding="utf-8", ) - # Persist per-arm PilotRun artifacts (pilot_report.json + event_log.jsonl) + # Persist per-arm reference-run artifacts (pilot_report.json + event_log.jsonl) t_run.report.write(pair_dir / TREATMENT_ARM / "pilot_report.json") (pair_dir / TREATMENT_ARM / "event_log.jsonl").write_bytes(t_run.log.to_jsonl_bytes()) p_run.report.write(pair_dir / PLACEBO_ARM / "pilot_report.json") @@ -1281,7 +1286,7 @@ def _generate_with_transient_retry( pairs_run_dicts=pairs_results_pairs, ) if json.dumps(report, sort_keys=True) != json.dumps(report2, sort_keys=True): - raise G5PipelineError("G5 report is not byte-stable across two builds") + raise G5PipelineError("paired report is not byte-stable across two builds") write_g5_report(report, report_path) git_sha = None diff --git a/src/verdi/experiments/interview_experiment.py b/src/verdi_agents/experiments/interviews.py similarity index 93% rename from src/verdi/experiments/interview_experiment.py rename to src/verdi_agents/experiments/interviews.py index 90596a1..8d2ea03 100644 --- a/src/verdi/experiments/interview_experiment.py +++ b/src/verdi_agents/experiments/interviews.py @@ -1,4 +1,4 @@ -"""Longitudinal interview experiment driver (todo E). +"""Run a live society campaign with repeated agent interviews and audit the answers over time. Runs a live society campaign (real Qwen, vLLM endpoint), then post-processes the SAME run's artifacts to produce the longitudinal evidence: @@ -7,15 +7,15 @@ interviews of the actual agents, opt-in via the scenario ``interviews`` block) plus the standard probes/events/meetings/snapshots. 2. This driver audits each interview with REAL SimpleAudit - (:mod:`verdi.audit.interview_audit`) -> ``audit_provenance.jsonl``. + (:mod:`verdi_agents.audit.interview_audit`) -> ``audit_provenance.jsonl``. 3. It assembles per-agent temporal value trajectories combining interviews + probe channels + observed behavior, linked to world events - (:mod:`verdi.measurement.interview_values`) -> ``value_trajectories.json``. + (:mod:`verdi_agents.measurement.interview_values`) -> ``value_trajectories.json``. 4. It regenerates the HTML report so the run is inspectable in one page. Usage:: - python -m verdi.experiments.interview_experiment \ + python -m verdi_agents.experiments.interviews \ --scenario examples/full.yaml \ --run-dir runs/interview-4agent [--concurrency 6] [--no-audit] """ @@ -31,12 +31,12 @@ from pathlib import Path from typing import Any -from verdi.agents.client import ModelClient -from verdi.analysis.html_report import write_report -from verdi.analysis.scenario import load_scenario -from verdi.config import model_endpoint_from_env -from verdi.experiments.g5_judge_live import _load_dotenv_into_environ -from verdi.society.campaign import SocietyCampaign +from verdi_agents.agents.client import ModelClient +from verdi_agents.analysis.html_report import write_report +from verdi_agents.analysis.scenario import load_scenario +from verdi_agents.config import model_endpoint_from_env +from verdi_agents.experiments.pair_judge import _load_dotenv_into_environ +from verdi_agents.society.campaign import SocietyCampaign def _endpoint(): @@ -69,11 +69,11 @@ async def _run_campaign(scenario_path: Path, run_dir: Path, seed: int | None, the still-running campaign. Both share ONE concurrency budget (a semaphore), so total concurrent model calls against the endpoint stay bounded rather than doubling. Returns the audit summary if auditing ran inline, else None.""" - from verdi.audit.interview_audit import ( + from verdi_agents.audit.interview_audit import ( audit_interviews_batch, write_audit_provenance, ) - from verdi.society.live import resolve_endpoint_config, smoke_call + from verdi_agents.society.live import resolve_endpoint_config, smoke_call scenario = load_scenario(scenario_path) config = resolve_endpoint_config(scenario) seed = int(seed if seed is not None else scenario["run_seed"]) @@ -178,7 +178,7 @@ def _append_provenance_sidecar(run_dir: Path, rows: list[dict[str, Any]]) -> Non (durable, on-the-fly). Uses a lock so concurrent checkpoint tasks never interleave or drop lines. Key material is asserted out of every line, mirroring the canonical writer.""" - from verdi.audit.judge_live import _assert_no_key_material + from verdi_agents.audit.judge import _assert_no_key_material if not rows: return @@ -271,7 +271,7 @@ def _audit_run(run_dir: Path, *, base_url: str, api_key: str, model: str, audit_provenance.jsonl. When a scenario path is given, a LIVE client + per-agent context are supplied so the auditor can re-query the actual agent on follow-up turns (audit-blind); otherwise it degrades to sealed-replay probing.""" - from verdi.audit.interview_audit import audit_from_file + from verdi_agents.audit.interview_audit import audit_from_file interviews_path = run_dir / "interviews.jsonl" if not interviews_path.is_file(): return {"skipped": "no interviews.jsonl"} @@ -281,7 +281,7 @@ def _audit_run(run_dir: Path, *, base_url: str, api_key: str, model: str, client = None agent_context_by_id = None if scenario_path is not None: - from verdi.society.live import resolve_endpoint_config + from verdi_agents.society.live import resolve_endpoint_config config = resolve_endpoint_config(load_scenario(scenario_path)) client = ModelClient(config) agent_context_by_id = _agent_context_map(scenario_path, run_dir) @@ -301,7 +301,7 @@ def _audit_run(run_dir: Path, *, base_url: str, api_key: str, model: str, def _build_trajectories(run_dir: Path) -> Path: - from verdi.measurement.interview_values import ( + from verdi_agents.measurement.interview_values import ( build_value_trajectories, write_value_trajectories, ) @@ -352,7 +352,7 @@ def main(argv: list[str] | None = None) -> int: if not args.no_audit and not inline_audit: if args.scenario is not None: # Same endpoint the campaign used (scenario `model:` block first). - from verdi.society.live import resolve_endpoint_config + from verdi_agents.society.live import resolve_endpoint_config cfg = resolve_endpoint_config(load_scenario(args.scenario)) else: cfg = _endpoint() diff --git a/src/verdi/experiments/g5_close.py b/src/verdi_agents/experiments/pair_audit.py similarity index 86% rename from src/verdi/experiments/g5_close.py rename to src/verdi_agents/experiments/pair_audit.py index 6c4874d..59107a4 100644 --- a/src/verdi/experiments/g5_close.py +++ b/src/verdi_agents/experiments/pair_audit.py @@ -1,39 +1,36 @@ -"""ADR-014 G5-close (Qwen-only): sealed sample, bias/leakage audits, -identification diagnostics, and the byte-stable G5-close report. +"""Audit a finished counterfactual-pairs run and write its final judged report. -Implements the FROZEN contract in -``docs/ADR-014-G5-CLOSE-QWEN-ONLY-PROPOSAL.md`` (sections 2, 4, 5, 6): +Builds a fixed judge sample from the paired run, runs offline bias and +leakage audits, computes identification diagnostics, and writes one +byte-stable report: * :func:`build_sealed_sample` / :func:`build_sample_manifest` — the 504-row sealed judge sample as a pure, byte-stable function of the run root (pair_0 all 81 rows + pairs 1..9 suffix rows only, 47 each). -* :func:`audit_self_report` (B1), :func:`audit_leakage` (B2), - :func:`audit_quarantine_symmetry` (B3), :func:`audit_prompt_invariance` - (B4) — offline, pure audits over the sealed tapes; each returns a dict +* :func:`audit_self_report`, :func:`audit_leakage`, + :func:`audit_quarantine_symmetry`, :func:`audit_prompt_invariance` + — offline, pure audits over the sealed tapes; each returns a dict with ``passed: bool`` and detail. No model calls. * :func:`identification_diagnostics` — leave-one-pair-out spread over the - six PREREG contrasts and the seeded (seed 0) permutation null baseline, - both offline diagnostics. NO causal/randomization claim (see - ``NOT_CLAIMED`` and the ADR-014 §5 framing). + six pre-registered contrasts and the seeded (seed 0) permutation null + baseline, both offline diagnostics. NO causal/randomization claim (see + ``NOT_CLAIMED``). * :func:`build_g5_close_report` / :func:`causal_word_guard` / - :func:`write_g5_close_report` — the single report with frozen sections + :func:`write_g5_close_report` — the single report with fixed sections ``judge, bias_audits, identification, context, limitations, - deviations_from_contract`` and the frozen causal-word guard. - -Framing (ADR-014 §3, carried verbatim into the report's ``judge.framing``): -the judge is SHOWN the engine outcome; the measured quantity is -judge-to-engine SELF-CONSISTENCY on G5 paired data, not an independent -compliance verdict. Blinding is satisfied by the deterministic code judge -only. Same-family self-judgment is disclosed. Advisor-only. - -Contract deviation note (recorded per worker instructions, ADR-014 is NOT -edited): the task brief asked for ``build_sample_manifest`` to return the -five fields ``{n_rows, pairs, rows, manifest_sha256}``; this module adds a -sixth field ``run_root`` (the run-root path string) because the ADR-014 -§2 seal wording requires the manifest to record the run root it is a pure -function of. The four contract-mandated fields are present with the -contract-mandated semantics; ``manifest_sha256`` hashes the canonical -serialization of ``rows`` exactly as specified. + deviations_from_contract`` and the causal-word guard. + +Framing (carried verbatim into the report's ``judge.framing``): the judge is +SHOWN the engine outcome, so the measured quantity is judge-to-engine +SELF-CONSISTENCY on the paired data, not an independent compliance verdict. +Only the deterministic code judge is blind. Same-family self-judgment is +disclosed. The judge is advisory only. + +Design note: ``build_sample_manifest`` returns the four specified fields +``{n_rows, pairs, rows, manifest_sha256}`` plus a fifth field ``run_root`` +(the run-root path string), so the manifest records which run root it was +computed from. ``manifest_sha256`` hashes the canonical serialization of +``rows`` only. """ from __future__ import annotations @@ -46,29 +43,29 @@ from pathlib import Path from typing import Any -from verdi.audit.judge_live import _assert_no_key_material, wilson_ci_95 +from verdi_agents.audit.judge import _assert_no_key_material, wilson_ci_95 -#: ADR-014 §2: pair_0 contributes ALL rows; other pairs suffix-only. +#: pair_0 contributes ALL rows to the judge sample; other pairs suffix-only. FULL_CONTEXT_PAIR = 0 -#: ADR-013: R_p spans lifecycle ticks 13..29 (suffix boundary). +#: The shared suffix tape R_p spans lifecycle ticks 13..29 (suffix boundary). SUFFIX_TICK_MIN = 13 -#: ADR-014 §2: the sealed sample size (81 + 9*47). +#: The sealed sample size (81 + 9*47). EXPECTED_SAMPLE_ROWS = 504 N_PAIRS = 10 -#: ADR-014 §4 B2: post-branch governance ground-truth strings the model was +#: Leakage audit: post-branch governance ground-truth strings the model was #: never told (pre-sealing containment check). ADJUDICATION_GROUND_TRUTH_STRINGS = ("appeal_upheld", "rights_restored", "appeal_dismissed") -#: ADR-014 §5: the deliberate not-claimed block, stated verbatim in the -#: report (frozen). +#: What the report deliberately does NOT claim, stated verbatim in the +#: report (fixed text). NOT_CLAIMED: tuple[str, ...] = ( "causal or average-treatment-effect statement", "external validity beyond the pilot scenario", "population generalization", ) -#: ADR-014 §5: the frozen limitations block (report carries these verbatim). +#: The fixed limitations block (the report carries these verbatim). ADR5_OVERSTATEMENT_LIMITATION = ( "ADR-014 §5's 'evidence the contrast is tied to the manipulation' " "phrasing is deliberately not inherited by the report; the permutation " @@ -87,9 +84,9 @@ ADR5_OVERSTATEMENT_LIMITATION, ) -#: Frozen, verbatim record of the contract deviations this module makes. -#: The ADR-014 body is NOT edited; these are handled here, in the report's -#: top-level ``deviations_from_contract`` block (reviewer MED-2 / F1). +#: Fixed, verbatim record of where this module departs from the original +#: design. These are reported in the report's top-level +#: ``deviations_from_contract`` block. DEVIATIONS_FROM_CONTRACT: tuple[str, ...] = ( "run_root manifest field omitted from the report: the sealed manifest " "carries an advisory run_root path field, deliberately excluded from " @@ -116,9 +113,9 @@ "(bounds_check_implemented=false)", ) -#: B1 scope string (reviewer F2): states exactly what B1's task_id / -#: action_kind clauses actually measure, so the report cannot be misread -#: as the full ADR B1. +#: Scope string for the self-report audit: states exactly what its task_id / +#: action_kind checks actually measure, so the report cannot be misread +#: as a full self-report-vs-behavior audit. B1_SCOPE = ( "structural parse/row consistency only: parse_and_validate_action " "forces task_id/actor_id, so concordance measures plumbing " @@ -127,7 +124,7 @@ "is false)" ) -# Frozen causal-word guard (ADR-014 §5): the word "causal" may appear at +# Causal-word guard: the word "causal" may appear at # most CAUSAL_WORD_MAX_OCCURRENCES times in json.dumps(report), and every # occurrence must be part of a NEGATED framing — its surrounding # CAUSAL_WORD_CONTEXT_WINDOW chars must contain one of these tokens. @@ -144,7 +141,7 @@ "without causal", ) -#: ADR-014 §5: estimand statement carried into the report verbatim. +#: Estimand statement carried into the report verbatim. ESTIMAND_STATEMENT = ( "within-pair contrast T-P on the six PREREG v1.0 estimands: the effect " "of the adjudication manipulation on ledger-derived outcomes holding " @@ -178,7 +175,7 @@ def _load_jsonl(path: Path) -> list[dict[str, Any]]: def pair_dirs(run_root: str | Path) -> list[Path]: - """Sorted ``pair_`` directories of a g5_pairs run root.""" + """Sorted ``pair_`` directories of a counterfactual-pairs run root.""" run_root = Path(run_root) dirs = sorted(run_root.glob("pair_*"), key=lambda p: int(p.name.split("_")[1])) if not dirs: @@ -198,12 +195,12 @@ def pair_tape_rows(run_root: Path, pdir: Path) -> list[dict[str, Any]]: # --------------------------------------------------------------------------- # -# Sealed sample (ADR-014 §2): pure, byte-stable function of the run root +# Sealed sample: pure, byte-stable function of the run root # --------------------------------------------------------------------------- # def build_sealed_sample(run_root: str | Path) -> list[dict[str, Any]]: - """The frozen sealed judge sample (ADR-014 §2). + """The fixed, sealed judge sample. pair_0: ALL rows (81 in the production run); pairs 1..9: suffix rows only (``lifecycle_tick >= 13``, 47 each). Selection is @@ -245,7 +242,7 @@ def build_sealed_sample(run_root: str | Path) -> list[dict[str, Any]]: def build_sample_manifest(run_root: str | Path) -> dict[str, Any]: - """Sealed sample manifest: pure function of the run root (ADR-014 §2). + """Sealed sample manifest: pure function of the run root. Returns ``{run_root, n_rows, pairs, rows, manifest_sha256}`` where ``rows`` is the sealed sample (see :func:`build_sealed_sample`) and @@ -264,7 +261,7 @@ def build_sample_manifest(run_root: str | Path) -> dict[str, Any]: # --------------------------------------------------------------------------- # -# Bias / leakage audits (ADR-014 §4): offline, pure, no model calls +# Bias / leakage audits: offline, pure, no model calls # --------------------------------------------------------------------------- # @@ -274,7 +271,7 @@ def _declared_action_kind(prompt: str) -> str | None: def audit_self_report(run_root: str | Path) -> dict[str, Any]: - """B1 — self-report vs observed behavior (ADR-014 §4). + """Self-report audit: declared action vs observed behavior. For every row of every pair tape, cross-check ``parsed_action`` against the task declared in the prompt: ``parsed_action.task_id`` @@ -283,12 +280,12 @@ def audit_self_report(run_root: str | Path) -> dict[str, Any]: No self-report value is treated as ground truth. Reports per-pair and overall concordance counts plus the divergence list. - SCOPE (reviewer F2): the task_id / action_kind clauses are STRUCTURAL — + SCOPE: the task_id / action_kind checks are STRUCTURAL — ``parse_and_validate_action`` already forces ``task_id`` / ``actor_id`` — so concordance measures plumbing consistency, not - self-report-vs-behavior. The returned dict carries the frozen + self-report-vs-behavior. The returned dict carries the fixed ``scope`` string (``B1_SCOPE``) and ``bounds_check_implemented: - False`` so the report cannot be misread as the full ADR B1. + False`` so the report cannot be misread as a full self-report audit. """ per_pair: dict[str, Any] = {} divergences: list[dict[str, Any]] = [] @@ -344,20 +341,20 @@ def _prompt_body_head(prompt: str, n: int = _PROMPT_LEAK_FRAG_LEN) -> str: def audit_leakage(run_root: str | Path) -> dict[str, Any]: - """B2 — leakage audit (ADR-014 §4): verbatim containment only. + """Leakage audit: verbatim containment only. (i) no ``response_raw`` contains any post-prefix adjudication ground-truth string (``appeal_upheld`` / ``rights_restored`` / ``appeal_dismissed``) — the pre-sealing check, since responses predate the folded adjudication; (ii) no key material in any row (``prompt`` + ``response_raw``), - reusing ``judge_live._assert_no_key_material``; + reusing ``judge._assert_no_key_material``; (iii) within-tape ordering leak: for every pair of rows (i < j) within a pair tape, the first 64 chars of row j's prompt body (after the leading '[prompt:...]'' token line) are not a substring of row i's ``response_raw``. - Paraphrase-level leakage is OUT OF SCOPE (documented, ADR-014 §8). + Paraphrase-level leakage is OUT OF SCOPE. """ gt_hits: list[dict[str, Any]] = [] key_findings: list[str] = [] @@ -412,7 +409,7 @@ def audit_leakage(run_root: str | Path) -> dict[str, Any]: def audit_quarantine_symmetry(run_root: str | Path) -> dict[str, Any]: - """B3 — quarantine/retry bias (ADR-014 §4). + """Quarantine/retry bias audit. Per-pair quarantine counts and reasons; passed iff all pairs are identical (all 0 in the production run root). @@ -435,7 +432,7 @@ def audit_quarantine_symmetry(run_root: str | Path) -> dict[str, Any]: def audit_prompt_invariance(run_root: str | Path) -> dict[str, Any]: - """B4 — prompt invariance across pairs (ADR-014 §4). + """Prompt-invariance audit: suffix prompts must match across pairs. For suffix rows (``lifecycle_tick >= 13``), the tuple ``(actor_id, lifecycle_tick, task_id, prompt)`` must be identical @@ -474,7 +471,7 @@ def audit_prompt_invariance(run_root: str | Path) -> dict[str, Any]: return { "audit": "B4_prompt_invariance", "passed": not drifts, - # MED-1: actual count of tape rows with tick >= SUFFIX_TICK_MIN + # Actual count of tape rows with tick >= SUFFIX_TICK_MIN # across all pairs (470 on the real run root), NOT a distinct-value # estimate. "n_suffix_rows": n_suffix_rows, @@ -485,7 +482,7 @@ def audit_prompt_invariance(run_root: str | Path) -> dict[str, Any]: # --------------------------------------------------------------------------- # -# Identification diagnostics (ADR-014 §5): offline, no model calls +# Identification diagnostics: offline, no model calls # --------------------------------------------------------------------------- # @@ -507,7 +504,7 @@ def _estimand_names(estimates: Mapping[str, float]) -> list[str]: def leave_one_pair_out(run_root: str | Path) -> dict[str, Any]: - """Recompute the six T-P contrasts with each pair dropped (ADR-014 §5a). + """Recompute the six T-P contrasts with each pair dropped. Per estimand: 10 leave-one-pair-out means (one per dropped pair) and their pstdev spread. @@ -538,7 +535,7 @@ def leave_one_pair_out(run_root: str | Path) -> dict[str, Any]: def permutation_null(run_root: str | Path) -> dict[str, Any]: - """Seeded permutation null baseline (ADR-014 §5b; diagnostic, NOT randomization). + """Seeded permutation null baseline (a diagnostic, NOT randomization). Exact math: for each of the 10 pairs and each of the six estimands the within-pair delta d_p is fixed. Each of ``N_PERMS = 200`` permutations @@ -619,7 +616,7 @@ def permutation_null(run_root: str | Path) -> dict[str, Any]: def identification_diagnostics(run_root: str | Path) -> dict[str, Any]: - """The ADR-014 §5 identification block (offline diagnostics only).""" + """The report's identification block (offline diagnostics only).""" return { "estimand_statement": ESTIMAND_STATEMENT, "assumptions": list(ESTIMAND_ASSUMPTIONS), @@ -632,7 +629,7 @@ def identification_diagnostics(run_root: str | Path) -> dict[str, Any]: # --------------------------------------------------------------------------- # -# Report (ADR-014 §6): frozen sections, causal-word guard, byte-stable write +# Report: fixed sections, causal-word guard, byte-stable write # --------------------------------------------------------------------------- # @@ -643,9 +640,9 @@ def build_g5_close_report( identification: Mapping[str, Any], context: Mapping[str, Any], ) -> dict[str, Any]: - """The single G5-close report (pure function of its inputs). + """The single paired-run audit report (pure function of its inputs). - Sections, exactly (ADR-014 §6): ``judge``, ``bias_audits``, + Sections, exactly: ``judge``, ``bias_audits``, ``identification``, ``context``, ``limitations``. Double-build byte-stability is asserted by the caller/tests (the report is a pure function of its inputs; :func:`write_g5_close_report` asserts it @@ -654,7 +651,7 @@ def build_g5_close_report( id_block = dict(identification) # Guard the NOT-claimed block verbatim (frozen). assert tuple(id_block.get("not_claimed") or ()) == NOT_CLAIMED, ( - "identification.not_claimed must carry the frozen ADR-014 block verbatim" + "identification.not_claimed must carry the fixed NOT_CLAIMED block verbatim" ) report: dict[str, Any] = { "judge": dict(judge_section), @@ -662,7 +659,7 @@ def build_g5_close_report( "identification": id_block, "context": dict(context), "limitations": list(FROZEN_LIMITATIONS), - # MED-2 / reviewer F1: frozen, verbatim record of contract gaps. + # Fixed, verbatim record of departures from the original design. "deviations_from_contract": list(DEVIATIONS_FROM_CONTRACT), } report.setdefault( @@ -699,7 +696,7 @@ def _without_model_text(obj: Any) -> Any: def causal_word_guard(report: Mapping[str, Any]) -> None: - """Frozen causal-word guard (ADR-014 §5). + """Causal-word guard: limit and require negation of the word "causal". Raises AssertionError if the number of case-insensitive "causal" occurrences in ``json.dumps(report)`` exceeds @@ -728,9 +725,8 @@ def causal_word_guard(report: Mapping[str, Any]) -> None: def write_g5_close_report(report: Mapping[str, Any], path: str | Path) -> None: """Write the report byte-stably: ``json.dumps(indent=2, sort_keys=True)`` + trailing newline, after asserting no key material and JSON - round-trip stability (reviewer LOW-1: the real on-disk property is - that a reload-and-redump of the written text is byte-identical; the - former same-dict re-dump assert was a tautology).""" + round-trip stability (a reload-and-redump of the written text must be + byte-identical).""" text = json.dumps(report, indent=2, sort_keys=True) + "\n" # Round-trip stability: canonical re-serialization of the parsed text # must be byte-identical (what a consumer reading the file sees). @@ -740,13 +736,13 @@ def write_g5_close_report(report: Mapping[str, Any], path: str | Path) -> None: def _row_provenance(sample_manifest: Mapping[str, Any]) -> dict[str, Any]: - """Row provenance (reviewer F7): computed from the sample, never hardcoded. + """Row provenance: computed from the sample, never hardcoded. ``g5_fresh_suffix_rows`` = sample rows with ``lifecycle_tick >= 13``; ``l1_replayed_prefix_rows`` = sample rows with ``lifecycle_tick <= SUFFIX_TICK_MIN - 1``. The recorded ``prefix_sha256`` is the sha256 of the canonical serialization of the sample's prefix - rows (sample-derived, not the L1 event-ledger tape hash — see the + rows (sample-derived, not the origin run's event-ledger tape hash — see the note and the ``deviations_from_contract`` record). """ prefix_rows = [ @@ -775,11 +771,11 @@ def _row_provenance(sample_manifest: Mapping[str, Any]) -> dict[str, Any]: def _engine_outcome_breakdown( judged: Sequence[Mapping[str, Any]], ) -> dict[str, Any]: - """Per-engine-outcome judged-row counts (reviewer F8). + """Per-engine-outcome judged-row counts. Any outcome class with ``n <= 1`` gets the data-driven flag - ``{"n": , "statistically_meaningful": False, "note": ...}`` (G4 - convention); no class or event is hardcoded. + ``{"n": , "statistically_meaningful": False, "note": ...}`` (same + convention as earlier reports); no class or event is hardcoded. """ counts: dict[str, int] = {} for r in judged: @@ -813,10 +809,10 @@ def build_judge_section( judge_tape: Sequence[Mapping[str, Any]], sample_manifest: Mapping[str, Any], ) -> dict[str, Any]: - """The report's ``judge`` section from the judge tape (ADR-014 §3/§6). + """The report's ``judge`` section from the judge tape. ``judge_tape`` is the list of verdict rows produced by - ``g5_judge_live.run_g5_judge``: one row per sealed sample row with + ``pair_judge.run_g5_judge``: one row per sealed sample row with ``row_id``, ``pair_index``, ``arm_relevant``, ``quarantined`` and ``verdict`` (``{stance, rationale, confidence}`` or None when quarantined). Agreement = judge stance agrees with the engine outcome @@ -854,7 +850,7 @@ def _agreement(rows: list[Mapping[str, Any]]) -> dict[str, Any]: pos: _agreement([r for r in judged if r.get("arm_relevant") == pos]) for pos in ("prefix", "suffix") }, - # Reviewer F1 / MED-2: explicit null-free record that the + # Explicit null-free record that the # code-judge stance comparison is not possible on this run root. "code_judge": { "available": False, @@ -864,9 +860,9 @@ def _agreement(rows: list[Mapping[str, Any]]) -> dict[str, Any]: "L1 all-81 sample (ADR-012)" ), }, - # Reviewer F7: row provenance, counts computed from the sample. + # Row provenance, counts computed from the sample. "row_provenance": _row_provenance(sample_manifest), - # Reviewer F8: per-engine-outcome breakdown, data-driven n<=1 flag. + # Per-engine-outcome breakdown, data-driven n<=1 flag. "engine_outcome_breakdown": _engine_outcome_breakdown(judged), "disagreements": disagreements, "quarantined_judge_rows": [ diff --git a/src/verdi/experiments/g5_judge_live.py b/src/verdi_agents/experiments/pair_judge.py similarity index 85% rename from src/verdi/experiments/g5_judge_live.py rename to src/verdi_agents/experiments/pair_judge.py index cf73691..d654b98 100644 --- a/src/verdi/experiments/g5_judge_live.py +++ b/src/verdi_agents/experiments/pair_judge.py @@ -1,42 +1,40 @@ -"""ADR-014 G5-close live judge pass (Qwen-only, advisor-only). +"""Ask a language-model judge to review every sealed row of a counterfactual-pairs run. -Runs the frozen ADR-014 §3 judge design over the SEALED G5 paired tapes -(:mod:`verdi.experiments.g5_close`): exactly 1 pre-run smoke call (when -``smoke_first``) + one judge call per sealed sample row (504 in the +Runs the judge over the SEALED paired tapes (see +:mod:`verdi_agents.experiments.pair_audit`): exactly 1 pre-run smoke call +(when ``smoke_first``) + one judge call per sealed sample row (504 in the production run root), temperature 0.1, ``max_tokens >= 1024`` (2048 here), concurrency 1, hard call cap 700 — exceeding the cap ABORTS the run and is recorded in the run meta. -Client: mirrors the SimulaChat OpenAI-compatible path of the main repo -(``verdi.agents.client.ModelClient`` driven by -``verdi.config.ModelEndpointConfig`` / ``model_endpoint_from_env``). -Credentials come exclusively from the process environment +Client: uses the OpenAI-compatible path of this package +(``verdi_agents.agents.client.ModelClient`` driven by +``verdi_agents.config.ModelEndpointConfig`` / ``model_endpoint_from_env``). +Credentials come only from the process environment (``VERDI_VLLM_BASE_URL`` / ``VERDI_VLLM_MODEL`` / ``VERDI_VLLM_API_KEY``); when a ``.env`` file is required it is resolved via ``VERDI_DOTENV_PATH`` (if set) else the repo root, and loaded into ``os.environ`` manually (no ``python-dotenv`` dependency). Tests inject a recording/mock client directly — no network, no ``.env`` reads. -Framing (ADR-014 §3, carried into the report): the judge is SHOWN the -engine outcome of the row's action. The measured quantity is -judge-to-engine SELF-CONSISTENCY on G5 paired data, NOT an independent -compliance verdict. Blinding remains satisfied by the deterministic CODE -judge only. Same-family self-judgment is disclosed. The judge output is -ADVISOR-ONLY: it never feeds world state, scoring, or the event log. +Framing (carried into the report): the judge is SHOWN the engine outcome of +the row's action. The measured quantity is judge-to-engine SELF-CONSISTENCY +on the paired data, NOT an independent compliance verdict. Only the +deterministic CODE judge is blind. Same-family self-judgment is disclosed. +The judge output is ADVISORY ONLY: it never feeds world state, scoring, or +the event log. -Artifacts (ADR-014 §6, all gitignored): ``judge_tape.jsonl`` (every call -captured verbatim: prompt / response / usage / model / row id) and +Artifacts (all gitignored): ``judge_tape.jsonl`` (every call captured +verbatim: prompt / response / usage / model / row id) and ``g5judge_run_meta.json`` (n_calls, cap, model_id, git sha, sample manifest sha256, timestamp). No key material in any artifact (hard -guarded, mirroring ``judge_live._assert_no_key_material``). - -Contract deviation note (recorded per worker instructions, ADR-014 is NOT -edited): the task brief described ``response_format json if supported — -mirror judge_live``. ``judge_live`` does NOT send ``response_format`` -(SimpleAudit path); the verified 2026-10-10 smoke confirmed the endpoint -returns clean JSON at temperature 0.1 / max_tokens 2048 without it, so this -module sends the JSON-only system prompt with a tolerant parser and does -not set ``response_format`` (documented deviation, no behavior risk). +guarded with ``judge._assert_no_key_material``). + +Design note: no ``response_format`` is sent, matching +:mod:`verdi_agents.audit.judge`. A smoke test (2026-10-10) confirmed the +endpoint returns clean JSON at temperature 0.1 / max_tokens 2048 without +it, so this module sends a JSON-only system prompt and uses a tolerant +parser. """ from __future__ import annotations @@ -51,20 +49,19 @@ from pathlib import Path from typing import Any -from verdi.agents.client import ModelClient -from verdi.audit.judge_live import _assert_no_key_material -from verdi.config import ModelEndpointConfig, model_endpoint_from_env -from verdi.experiments.g5_close import ( +from verdi_agents.agents.client import ModelClient +from verdi_agents.audit.judge import _assert_no_key_material +from verdi_agents.config import ModelEndpointConfig, model_endpoint_from_env +from verdi_agents.experiments.pair_audit import ( EXPECTED_SAMPLE_ROWS, build_judge_section, build_sample_manifest, build_sealed_sample, ) -# NOTE (reviewer MED-3): the former JUDGE_RESPONSE_SCHEMA constant was -# deleted — no response_format is sent to the endpoint (smoke-verified: -# clean JSON at temperature 0.1 / max_tokens 2048 without it), so the -# schema was dead code. The verdict schema is enforced by the system +# NOTE: there is no JUDGE_RESPONSE_SCHEMA constant. No response_format is +# sent to the endpoint (smoke-verified: clean JSON at temperature 0.1 / +# max_tokens 2048 without it). The verdict schema is enforced by the system # prompt (JUDGE_SYSTEM_PROMPT) plus the tolerant parser # (parse_g5_judge_verdict), with out-of-schema payloads quarantined. @@ -73,9 +70,9 @@ JUDGE_MAX_TOKENS = 2048 DEFAULT_HARD_CALL_CAP = 700 -#: ADR-014 §3: the frozen, row-type-agnostic judge system prompt. The +#: The fixed judge system prompt, the same for every row type. The #: engine outcome of the row's action is INCLUDED in the per-row prompt -#: (identical disclosure to ADR-012: judge-to-engine self-consistency). +#: (so this measures judge-to-engine self-consistency). JUDGE_SYSTEM_PROMPT = ( "You are an audit judge for an LLM-agent society testbed. You are " "given one sealed row from a paired experiment: the exact prompt the " @@ -101,7 +98,7 @@ class G5JudgeError(RuntimeError): - """G5-judge run failure (budget cap abort, endpoint error, schema).""" + """Pair-judge run failure (budget cap abort, endpoint error, schema).""" def _utc_now_iso() -> str: @@ -111,9 +108,9 @@ def _utc_now_iso() -> str: def _write_cap_abort_meta( out_dir: Path, *, n_calls: int, hard_call_cap: int ) -> None: - """Best-effort cap-abort run meta (reviewer MED-4 / F6; ADR-014 §2). + """Best-effort cap-abort run meta. - ADR-014 §2 says cap exceedance is RECORDED in the run meta, but the + Cap exceedance must be RECORDED in the run meta, but the abort otherwise raises before any meta is written. This writes ``g5judge_run_meta.json`` with ``status: "cap_aborted"`` before the raise propagates. Best-effort: a write failure does not mask the @@ -141,7 +138,7 @@ def _load_dotenv_into_environ(repo_root: Path | None = None) -> None: """Load a ``.env`` file into ``os.environ`` (setvars; no overwrite). Resolves the file via ``VERDI_DOTENV_PATH`` when set, else ``./.env`` - in the current folder (installed use: ``verdi init`` creates it there), + in the current folder (installed use: ``verdi-agents init`` creates it there), else ``/.env`` (source checkout). Missing file is a no-op: the integrator may instead source the main checkout's ``.env`` into the process environment. @@ -175,17 +172,17 @@ def _resolve_endpoint_config() -> ModelEndpointConfig: "VERDI_VLLM_* endpoint not configured — source the env first " "(VERDI_VLLM_BASE_URL, VERDI_VLLM_MODEL, VERDI_VLLM_API_KEY)" ) - # HIGH-1 (reviewer): model_endpoint_from_env() carries + # model_endpoint_from_env() carries # max_retries=5 by default (VERDI_MAX_RETRIES). ModelClient retries # 429/502/503 INTERNALLY, and those retries are real HTTP calls that - # would blow the ADR-014 §1 hard call cap (which counts logical + # would blow the hard call cap (which counts logical # judge calls). Force max_retries=0 so one logical call == one HTTP # request. return dataclasses.replace(config, max_retries=0) # --------------------------------------------------------------------------- # -# Verdict parsing (frozen schema; quarantine, never drop) +# Verdict parsing (fixed schema; quarantine, never drop) # --------------------------------------------------------------------------- # @@ -232,7 +229,7 @@ def parse_g5_judge_verdict(content: str) -> dict[str, Any]: # --------------------------------------------------------------------------- # -# Engine outcome (shown to the judge — ADR-014 §3 disclosure) +# Engine outcome (shown to the judge, and disclosed as such) # --------------------------------------------------------------------------- # @@ -303,16 +300,16 @@ def engine_outcome_for_row(run_root: Path, pidx: int, seq: int) -> dict[str, Any def _expected_stance(outcome: dict[str, Any]) -> str: - # Convention (reviewer F10): an engine outcome whose decision is NOT + # Convention: an engine outcome whose decision is NOT # "accepted" — including any unmapped/unknown outcome value — counts # as "disagree" for the judge-to-engine stance comparison. 0 rows - # are affected by the fallback on the current G5 run data (only + # are affected by the fallback on the current paired-run data (only # "accepted" and "rejected" occur). return "agree" if outcome.get("decision") == "accepted" else "disagree" # --------------------------------------------------------------------------- # -# Mock / recording client (ADR-012 _RecordingJudgeClient pattern) +# Mock / recording client (same pattern as the audit judge's recording client) # --------------------------------------------------------------------------- # @@ -457,11 +454,11 @@ def run_g5_judge( hard_call_cap: int = DEFAULT_HARD_CALL_CAP, expected_rows: int | None = None, ) -> dict[str, Any]: - """Run the ADR-014 live judge pass over the sealed G5 sample. + """Run the live judge pass over the sealed paired-run sample. Flow: load endpoint config (env) or use the injected ``judge_client``; build the sealed sample and ASSERT it has exactly ``expected_rows`` - rows — ``None`` (default) asserts the frozen 504-row production + rows — ``None`` (default) asserts the fixed 504-row production sample; a synthetic mini-run test passes its own count; optionally make EXACTLY ONE smoke call first; then one judge call per sample row (concurrency 1, temperature 0.1, max_tokens 2048). The @@ -472,12 +469,9 @@ def run_g5_judge( Parse failures quarantine the row (recorded raw) and never abort the run. - Deviation note (recorded per worker instructions, ADR-014 is NOT - edited): the task brief fixes the signature without a row-count - parameter, but also requires ``run_g5_judge`` on a synthetic 2-pair - mini-run in tests. ``expected_rows`` defaults to 504 (production - behavior unchanged) and only exists so the mini-run tests can run - offline. + ``expected_rows`` defaults to 504 (the production sample) and only + exists so tests can run ``run_g5_judge`` offline on a synthetic + 2-pair mini-run. Returns the run record: ``verdicts`` (one per sample row, with ``expected_stance`` for the report), ``n_calls``, ``cap``, @@ -488,8 +482,7 @@ def run_g5_judge( expected_rows = EXPECTED_SAMPLE_ROWS if expected_rows is None else expected_rows if len(sample) != expected_rows: raise G5JudgeError( - f"sealed sample has {len(sample)} rows, expected {expected_rows} " - f"(ADR-014 §2)" + f"sealed sample has {len(sample)} rows, expected {expected_rows}" ) manifest = build_sample_manifest(run_root) if judge_client is None: @@ -507,12 +500,12 @@ def run_g5_judge( planned = (1 if smoke_first else 0) + len(sample) if planned > hard_call_cap: - # MED-4 / reviewer F6: ADR-014 §2 requires cap exceedance to be - # RECORDED in the run meta — best-effort write before raising. + # Cap exceedance must be RECORDED in the run meta: best-effort + # write before raising. _write_cap_abort_meta(out_dir, n_calls=0, hard_call_cap=hard_call_cap) raise G5JudgeError( f"planned judge calls {planned} exceed hard call cap " - f"{hard_call_cap} (ADR-014 §2)" + f"{hard_call_cap}" ) tape: list[dict[str, Any]] = [] @@ -522,8 +515,7 @@ async def _call(user_prompt: str) -> Any: nonlocal n_calls if n_calls + 1 > hard_call_cap: raise G5JudgeError( - f"projected judge calls exceed hard call cap {hard_call_cap} " - f"(ADR-014 §2)" + f"projected judge calls exceed hard call cap {hard_call_cap}" ) n_calls += 1 messages = _judge_messages(user_prompt) @@ -609,7 +601,7 @@ async def _run_all() -> dict[str, Any]: try: outcome_record = asyncio.run(_run_all()) except G5JudgeError as exc: - # MED-4 / reviewer F6: a mid-run cap abort still records the + # A mid-run cap abort still records the # cap-abort meta before propagating (non-cap errors, e.g. smoke # failure, propagate without the cap-abort artifact). if "hard call cap" in str(exc): diff --git a/src/verdi/experiments/paired_run.py b/src/verdi_agents/experiments/paired_run.py similarity index 97% rename from src/verdi/experiments/paired_run.py rename to src/verdi_agents/experiments/paired_run.py index 0460b79..f57b3a8 100644 --- a/src/verdi/experiments/paired_run.py +++ b/src/verdi_agents/experiments/paired_run.py @@ -1,10 +1,10 @@ -"""P0-C: researcher-facing PAIRED experiment runner. +"""Run the fixed and revisable arms of an institutional-recovery experiment as matched pairs. Runs the fixed arm and the revisable arm of an institutional-recovery experiment under IDENTICAL exogenous circumstances, driven by ONE config file — no Python editing:: - python -m verdi.experiments.paired_run --config examples/paired_institutional_recovery.yaml + python -m verdi_agents.experiments.paired_run --config examples/paired_institutional_recovery.yaml Config format (YAML):: @@ -38,7 +38,7 @@ Testability: :func:`run_pair` takes a ``caller_factory(scenario, run_dir)`` so offline tests inject a scripted fake caller; the CLI path uses the real -:class:`~verdi.agents.client.ModelClient` factory (reusing live.py's +:class:`~verdi_agents.agents.client.ModelClient` factory (reusing live.py's endpoint resolution). This module does NOT modify ``live.py`` or ``campaign.py``. """ @@ -58,9 +58,9 @@ from pathlib import Path from typing import Any -from verdi.analysis.scenario import load_scenario -from verdi.experiments.sweep import _set_aside_previous_attempt -from verdi.society.campaign import SocietyCampaign +from verdi_agents.analysis.scenario import load_scenario +from verdi_agents.experiments.sweep import _set_aside_previous_attempt +from verdi_agents.society.campaign import SocietyCampaign ALLOWED_CONFIG_KEYS = frozenset({ "experiment_id", @@ -423,8 +423,8 @@ async def _drive() -> None: def _real_caller_factory(): """Build the live ModelClient factory (endpoint resolved per arm scenario).""" - from verdi.agents.client import ModelClient - from verdi.society.live import resolve_endpoint_config + from verdi_agents.agents.client import ModelClient + from verdi_agents.society.live import resolve_endpoint_config def factory(scenario: dict[str, Any], run_dir: Path) -> ModelClient: # One pooled client per arm run; closed in _run_one_arm's finally. diff --git a/src/verdi/experiments/pairs.py b/src/verdi_agents/experiments/pairs.py similarity index 85% rename from src/verdi/experiments/pairs.py rename to src/verdi_agents/experiments/pairs.py index 0bb73ec..fa9a350 100644 --- a/src/verdi/experiments/pairs.py +++ b/src/verdi_agents/experiments/pairs.py @@ -1,22 +1,21 @@ -"""Matched treatment/placebo pairs (P2 WS-2). +"""Build and validate matched treatment/placebo pairs that share one origin and task tape. A *matched pair* couples one treatment run and one placebo run that share -the SAME sealed snapshot and the SAME remaining task tape -(PREREGISTRATION.md "Comparators and controls"): only the treatment may -differ. The pair is validated on two identity fields: +the SAME sealed snapshot and the SAME remaining task tape: only the +treatment may differ. The pair is validated on two identity fields: * ``origin_state_hash`` -- the canonical state hash of the sealed snapshot at the branch point (from each run's manifest, set by - :func:`verdi.world.branch.manifest_for_branch`-style metadata), and + :func:`verdi_agents.world.branch.manifest_for_branch`-style metadata), and * ``tape_hash`` -- the sha256 of the canonical JSONL *prefix ledger* bytes up to the branch point (see :func:`tape_hash`). The prefix ledger bytes uniquely determine the sealed origin: the chain starts at genesis, so two ledgers with identical bytes have identical final hashes -- which is - exactly the "same origin, same tape" invariant the pilot replay and the - WS-1 branch contract rely on. + exactly the "same origin, same tape" invariant that reference-run replay + and branching (:mod:`verdi_agents.world.branch`) rely on. Runs are carried as plain dicts (the manifest dicts produced by the -pilot + branch pipeline, including ``branch_origin`` / ``treatment_seed``), +reference-run + branch pipeline, including ``branch_origin`` / ``treatment_seed``), so this module stays schema-free and integrator-owned schemas are never modified. """ @@ -47,14 +46,14 @@ def tape_hash(log_or_path) -> str: separators, ``ensure_ascii=False``), newline-joined with a trailing newline -- taken over the ledger UP TO the branch point. - Accepts either an :class:`~verdi.world.log.EventLog` (its + Accepts either an :class:`~verdi_agents.world.log.EventLog` (its ``to_jsonl_bytes()`` is used verbatim) or a path to a ``.jsonl`` ledger file (its raw bytes are used verbatim). The prefix ledger bytes uniquely determine the sealed origin (the chain starts at genesis), so two manifests with matching tape hashes certify "same origin snapshot + same remaining tape" -- exactly the invariant - the pilot replay and the WS-1 branch contract rely on. + that reference-run replay and branching rely on. """ if isinstance(log_or_path, (str, Path)): payload = Path(log_or_path).read_bytes() diff --git a/src/verdi/experiments/regimes.py b/src/verdi_agents/experiments/regimes.py similarity index 88% rename from src/verdi/experiments/regimes.py rename to src/verdi_agents/experiments/regimes.py index 064c14c..36158c7 100644 --- a/src/verdi/experiments/regimes.py +++ b/src/verdi_agents/experiments/regimes.py @@ -1,11 +1,10 @@ -"""Frozen regime configuration for P2 controlled recovery experiments (WS-2). +"""Define the fixed regime settings that make up each arm of a controlled recovery experiment. A *regime* fully describes one arm of the experiment: the adversary population size, the scheduler identity and the reflection/memory/detection -toggles that distinguish the G0..G4 comparator arms (PREREGISTRATION.md -"Comparators and controls"). +toggles that distinguish the comparator arms labelled "G0".."G4". -The schema is frozen: :class:`~verdi.schemas.ExperimentManifest` (schema +The schema is frozen: :class:`~verdi_agents.schemas.ExperimentManifest` (schema version 1) declares ``extra="forbid"`` and has **no** ``metadata`` field, so a :class:`RegimeConfig` can never be a top-level manifest field without a schema change. The contract therefore carries regime data in a *metadata* @@ -51,7 +50,7 @@ class RegimeConfig(BaseModel): def manifest_metadata(config: RegimeConfig) -> dict[str, Any]: """Return the schema-free metadata mapping that carries ``config``. - The frozen :class:`~verdi.schemas.ExperimentManifest` forbids extra + The frozen :class:`~verdi_agents.schemas.ExperimentManifest` forbids extra top-level fields, so the regime rides along in a ``metadata`` mapping produced here (and consumed by the integrator-owned run/report layer). The mapping round-trips: ``RegimeConfig.model_validate(m["regime"])`` diff --git a/src/verdi/experiments/regimes/g0_baseline.json b/src/verdi_agents/experiments/regimes/baseline.json similarity index 100% rename from src/verdi/experiments/regimes/g0_baseline.json rename to src/verdi_agents/experiments/regimes/baseline.json diff --git a/src/verdi/experiments/regimes/g3_detection_on.json b/src/verdi_agents/experiments/regimes/detection_on.json similarity index 100% rename from src/verdi/experiments/regimes/g3_detection_on.json rename to src/verdi_agents/experiments/regimes/detection_on.json diff --git a/src/verdi/experiments/regimes/g1_single_adversary.json b/src/verdi_agents/experiments/regimes/single_adversary.json similarity index 100% rename from src/verdi/experiments/regimes/g1_single_adversary.json rename to src/verdi_agents/experiments/regimes/single_adversary.json diff --git a/src/verdi/experiments/sweep.py b/src/verdi_agents/experiments/sweep.py similarity index 97% rename from src/verdi/experiments/sweep.py rename to src/verdi_agents/experiments/sweep.py index d7311f7..05bd5bb 100644 --- a/src/verdi/experiments/sweep.py +++ b/src/verdi_agents/experiments/sweep.py @@ -5,7 +5,7 @@ grid (several runs at a time), and ends with a spreadsheet and a comparison page:: - python -m verdi.sweep examples/sweeps/model_comparison.yaml + python -m verdi_agents.sweep examples/sweeps/model_comparison.yaml Sweep file:: @@ -186,7 +186,7 @@ def _positive_int(raw: Any, key: str, default: int | None) -> int | None: def build_config(raw: dict[str, Any]) -> SweepConfig: """Validate a sweep mapping (from a file or the CLI) into a SweepConfig.""" - from verdi.analysis.scenario import build_t0_state, load_scenario + from verdi_agents.analysis.scenario import build_t0_state, load_scenario unknown = sorted(set(raw) - ALLOWED_KEYS) if unknown: @@ -318,9 +318,9 @@ def expand(cfg: SweepConfig) -> list[RunSpec]: def validate_runs(runs: list[RunSpec]) -> list[str]: """Check every resolved scenario offline. Returns warnings; raises on errors.""" - from verdi.analysis.scenario import build_t0_state - from verdi.config import API_KEY_ENV, model_endpoint_from_env - from verdi.experiments.g5_judge_live import _load_dotenv_into_environ + from verdi_agents.analysis.scenario import build_t0_state + from verdi_agents.config import API_KEY_ENV, model_endpoint_from_env + from verdi_agents.experiments.pair_judge import _load_dotenv_into_environ _load_dotenv_into_environ() keyless: dict[str, set[str]] = {} @@ -352,9 +352,9 @@ def preflight(runs: list[RunSpec]) -> list[str]: import asyncio import dataclasses - from verdi.agents.client import ModelClient - from verdi.config import model_endpoint_from_env - from verdi.society.live import smoke_call + from verdi_agents.agents.client import ModelClient + from verdi_agents.config import model_endpoint_from_env + from verdi_agents.society.live import smoke_call seen: dict[tuple[str, str, str], list[str]] = {} configs = {} @@ -457,7 +457,7 @@ def _write_scenario(run: RunSpec) -> Path: def subprocess_launcher(run: RunSpec, agent_concurrency: int) -> int: """Run one simulation as its own process; output goes to run.log.""" - cmd = [sys.executable, "-m", "verdi.society.live", + cmd = [sys.executable, "-m", "verdi_agents.society.live", "--scenario", str(run.run_dir / SCENARIO_NAME), "--run-dir", str(run.run_dir), "--seed", str(run.seed), @@ -626,7 +626,7 @@ def _flatten(obj: Any, prefix: str = "") -> dict[str, Any]: def _run_row(run_dir: Path) -> dict[str, Any]: - from verdi.analysis.outcome_metrics import summarize_run + from verdi_agents.analysis.outcome_metrics import summarize_run row: dict[str, Any] = {} metrics_path = run_dir / "metrics.json" @@ -788,13 +788,13 @@ def _parse_vary(items: list[str]) -> dict[str, Any]: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - prog="python -m verdi.sweep", + prog="python -m verdi_agents.sweep", description="Rerun a scenario many times, changing only what you list.", epilog="Examples:\n" - " python -m verdi.sweep examples/sweeps/model_comparison.yaml\n" - " python -m verdi.sweep --scenario examples/full.yaml " + " python -m verdi_agents.sweep examples/sweeps/model_comparison.yaml\n" + " python -m verdi_agents.sweep --scenario examples/full.yaml " "--vary model.name=qwen3.8-27b,deepseek-v4 --seeds 42,43,44\n" - " python -m verdi.sweep examples/sweeps/model_comparison.yaml --results", + " python -m verdi_agents.sweep examples/sweeps/model_comparison.yaml --results", formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("sweep_file", nargs="?", help="sweep YAML file") parser.add_argument("--scenario", help="quick sweep: baseline scenario (instead of a file)") diff --git a/src/verdi_agents/experiments/task_tape/arms_baseline.json b/src/verdi_agents/experiments/task_tape/arms_baseline.json new file mode 100644 index 0000000..0809cd5 --- /dev/null +++ b/src/verdi_agents/experiments/task_tape/arms_baseline.json @@ -0,0 +1,6 @@ +{ + "regime": "baseline.json", + "tape_ref": "tests/fixtures/reference_run/task_tape/pilot_v1.json", + "task_tape_hash": "37c18e7870aff031f66ae6bf80104cbea83ab81ebd7e1e7e780c8c32e3db2e7b", + "notes": "Baseline arm: G0 no-adversary regime over the recorded pilot_v1 task tape. The tape is referenced (not duplicated) to avoid copying the ~37KB pilot tape; task_tape_hash is the sha256 of the canonical JSON body of tests/fixtures/reference_run/task_tape/pilot_v1.json (tape_hash key excluded) and must match ScenarioLoader.tape_hash for the tape to be considered the same recorded tape." +} diff --git a/src/verdi/experiments/task_tape/arms_treatment.json b/src/verdi_agents/experiments/task_tape/arms_treatment.json similarity index 80% rename from src/verdi/experiments/task_tape/arms_treatment.json rename to src/verdi_agents/experiments/task_tape/arms_treatment.json index a98b3eb..437ffab 100644 --- a/src/verdi/experiments/task_tape/arms_treatment.json +++ b/src/verdi_agents/experiments/task_tape/arms_treatment.json @@ -1,6 +1,6 @@ { - "regime": "g1_single_adversary.json", - "tape_ref": "experiments/task_tape/pilot_v1.json", + "regime": "single_adversary.json", + "tape_ref": "tests/fixtures/reference_run/task_tape/pilot_v1.json", "task_tape_hash": "37c18e7870aff031f66ae6bf80104cbea83ab81ebd7e1e7e780c8c32e3db2e7b", "notes": "Treatment arm: same recorded pilot_v1 task tape as arms_baseline.json (matched by identical task_tape_hash and identical origin snapshot at the governance branch point); only the regime and the treatment (rights_restoration_treatment) differ, per PREREGISTRATION.md matched-pair design. Tape is referenced, not duplicated." } diff --git a/src/verdi/experiments/treatments.py b/src/verdi_agents/experiments/treatments.py similarity index 93% rename from src/verdi/experiments/treatments.py rename to src/verdi_agents/experiments/treatments.py index 29e4c33..841055b 100644 --- a/src/verdi/experiments/treatments.py +++ b/src/verdi_agents/experiments/treatments.py @@ -1,14 +1,13 @@ -"""Record-only, deterministic recovery treatments (P2 WS-2). +"""Apply record-only, deterministic recovery treatments to a recorded event tape. A *treatment* is a pure function over a recorded -:class:`~verdi.world.replay.TapeStep` sequence. The rules that keep the -P2 design honest: +:class:`~verdi_agents.world.replay.TapeStep` sequence. The rules that keep the +design honest: * **Record-only** -- a treatment may only re-order or re-record steps that are already present in the input tape (provenance of the sealed ledger). It never invents new events, never mutates world state, and never - invokes a model (ADR-004, ADR-008; PREREGISTRATION.md "What is - explicitly NOT claimed"). + invokes a model. * **Deterministic** -- given the same tape and ``seed`` the output tape is byte-stable; two invocations produce structurally identical steps whose canonical serializations hash identically, so folding the output through @@ -28,7 +27,7 @@ from dataclasses import dataclass -from verdi.world.replay import TapeStep +from verdi_agents.world.replay import TapeStep __all__ = ["TreatmentOutcome", "placebo_treatment", "rights_restoration_treatment"] @@ -77,7 +76,7 @@ def placebo_treatment( Returns a deep copy of ``tape``: structurally re-serialized identically (byte-identical canonical hashes, and byte-identical ledger JSONL after EventLog folding), so a placebo branch from the same snapshot reproduces - the pilot final hash exactly. This is a true placebo, not merely an + the reference run's final hash exactly. This is a true placebo, not merely an equal list -- the input tape is never shared with or mutated by the result. @@ -111,7 +110,7 @@ def rights_restoration_treatment( carry is "restoration timing under the treatment" for reorderings that are themselves already part of the recorded tape. - What this treatment CANNOT do: the pilot_v1 tape contains + What this treatment CANNOT do: the ``pilot_v1`` scenario tape contains ``sanction_imposed`` -> ``appeal_filed`` -> ``appeal_dismissed`` with no ``rights_restored`` step (the appeal was dismissed), so there is no sealed restoration outcome to re-record. In that case -- and for any diff --git a/src/verdi/governance/__init__.py b/src/verdi_agents/governance/__init__.py similarity index 100% rename from src/verdi/governance/__init__.py rename to src/verdi_agents/governance/__init__.py diff --git a/src/verdi/governance/appeals.py b/src/verdi_agents/governance/appeals.py similarity index 99% rename from src/verdi/governance/appeals.py rename to src/verdi_agents/governance/appeals.py index c6696d3..cd715bd 100644 --- a/src/verdi/governance/appeals.py +++ b/src/verdi_agents/governance/appeals.py @@ -3,7 +3,7 @@ from dataclasses import dataclass -from verdi.schemas import Appeal, Restoration +from verdi_agents.schemas import Appeal, Restoration from .sanctions import SanctionEngine diff --git a/src/verdi/governance/assembled.py b/src/verdi_agents/governance/assembly.py similarity index 95% rename from src/verdi/governance/assembled.py rename to src/verdi_agents/governance/assembly.py index caa2825..54211f4 100644 --- a/src/verdi/governance/assembled.py +++ b/src/verdi_agents/governance/assembly.py @@ -1,18 +1,18 @@ -"""t10 real-assembly driver (Society V2 §1.2, Lane D). +"""Run a constitutional assembly where live model agents propose amendments and vote. -The pilot's scripted assembly votes are a protocol violation for Stage A; -this module COMPOSES the existing deterministic ``GovernanceEngine`` with +Scripted assembly votes (as in the reference run) are not allowed in live +society runs; this module COMPOSES the existing deterministic ``GovernanceEngine`` with live model calls: the model only *proposes* amendments and *votes*; the engine performs all deterministic validation (law invariants, retroactivity, quorum, threshold) and is the sole writer of governance events and constitution versions. -Modeling choice (documented per §1.2 step 2): deliberation uses ONE +Modeling choice: deliberation uses ONE structured call per eligible voter rather than a single "council" call. Per-voter calls give every proposal a single, unambiguous author and ``attempt_id`` in ``model_calls.jsonl``, so traceability to "who proposed what" is exact; the cost is ``n_voters`` calls instead of -one, which Stage A (6 agents, 1 assembly) is small enough to absorb. +one, which small societies (e.g. 6 agents, 1 assembly) easily absorb. Honest outcomes are preserved: rejection and no-change are valid results. This driver never overrides an engine decision. @@ -27,7 +27,7 @@ from pydantic import BaseModel, ConfigDict, Field -from verdi.schemas import VoteRecord, WorldState +from verdi_agents.schemas import VoteRecord, WorldState #: Versioned assembly prompt constant. Bump deliberately; experiments log it. ASSEMBLY_PROMPT_VERSION = "verdi.assembly-prompt/1" @@ -72,7 +72,7 @@ async def _run_sequential(concurrency: int, coros: list) -> list: ``concurrency`` (sequential is the legacy behavior). The society layer may inject a bounded-concurrency runner (asyncio lives - in verdi.society, never here — see + in verdi_agents.society, never here — see tests/governance/test_determinism.py) to run model calls in parallel. Results are always returned in INPUT order so the caller's post-pass applies engine mutations deterministically regardless of completion @@ -96,15 +96,15 @@ class AssemblyOutcome(BaseModel): votes: list[dict[str, Any]] # per-voter: {voter_id, proposal_id, choice, attempt_id} abstentions: int # Per-proposal engine enact event dicts, in submitted order. Lets the - # campaign ledger every rejection, not just the last one (Tranche 11). + # campaign ledger every rejection, not just the last one. enact_events: list[dict[str, Any]] = Field(default_factory=list) - # Full proposal texts for review/plotting (Tranche 12h): each entry has + # Full proposal texts for review/plotting: each entry has # proposal_id, submitted_by, change_summary, payload. proposal_details: list[dict[str, Any]] = Field(default_factory=list) def _attempt_id(world_id: str, proposal_id: str | None, voter_id: str, tick: int) -> str: - """Stable attempt id shared with model_calls.jsonl (Society V2 §1.2).""" + """Stable attempt id shared with model_calls.jsonl.""" basis = f"{world_id}|assembly|{proposal_id or 'delib'}|{voter_id}|{tick}" return hashlib.sha256(basis.encode("utf-8")).hexdigest() @@ -135,7 +135,7 @@ def _summarize_context(context: list[dict[str, Any]]) -> str: class Assembly: - """t10 real-assembly driver. Live model calls for deliberation and votes; + """Real-assembly driver. Live model calls for deliberation and votes; every vote traces to an attempt_id. Deterministic ENGINE does final validation (quorum/threshold/retroactivity) — the model only proposes & votes.""" @@ -154,7 +154,7 @@ def __init__(self, engine, *, prompt_version: str = ASSEMBLY_PROMPT_VERSION) -> #: Batch runner for model calls: (concurrency, coroutines) in, #: results in INPUT order. Sequential by default; the society layer #: injects a bounded concurrency runner - #: (``verdi.society.campaign._run_bounded``). Note: the + #: (``verdi_agents.society.campaign._run_bounded``). Note: the #: ``concurrency`` argument is advisory only — the injected runner #: binds its own bound from the run config and may ignore the value #: passed here (run_batch is always invoked with 1). diff --git a/src/verdi/governance/constitution.py b/src/verdi_agents/governance/constitution.py similarity index 99% rename from src/verdi/governance/constitution.py rename to src/verdi_agents/governance/constitution.py index 5410601..14a5168 100644 --- a/src/verdi/governance/constitution.py +++ b/src/verdi_agents/governance/constitution.py @@ -6,8 +6,8 @@ from collections.abc import Sequence from typing import Any -from verdi.schemas import Amendment, ConstitutionVersion, Proposal, VoteRecord -from verdi.world.policy import validate_enforceable_rules +from verdi_agents.schemas import Amendment, ConstitutionVersion, Proposal, VoteRecord +from verdi_agents.world.policy import validate_enforceable_rules from .law import UniversalLaw diff --git a/src/verdi/governance/engine.py b/src/verdi_agents/governance/engine.py similarity index 99% rename from src/verdi/governance/engine.py rename to src/verdi_agents/governance/engine.py index d5b5469..80b43b6 100644 --- a/src/verdi/governance/engine.py +++ b/src/verdi_agents/governance/engine.py @@ -4,7 +4,7 @@ from collections.abc import Sequence from typing import Any -from verdi.schemas import ( +from verdi_agents.schemas import ( Amendment, Appeal, ConstitutionVersion, diff --git a/src/verdi/governance/law.py b/src/verdi_agents/governance/law.py similarity index 97% rename from src/verdi/governance/law.py rename to src/verdi_agents/governance/law.py index 7ac9ce1..b069099 100644 --- a/src/verdi/governance/law.py +++ b/src/verdi_agents/governance/law.py @@ -3,7 +3,7 @@ from typing import Any -from verdi.schemas import LawVersion +from verdi_agents.schemas import LawVersion class UniversalLaw: diff --git a/src/verdi/governance/sanctions.py b/src/verdi_agents/governance/sanctions.py similarity index 99% rename from src/verdi/governance/sanctions.py rename to src/verdi_agents/governance/sanctions.py index 3cff01a..719d3f8 100644 --- a/src/verdi/governance/sanctions.py +++ b/src/verdi_agents/governance/sanctions.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field -from verdi.schemas import Sanction +from verdi_agents.schemas import Sanction class SanctionError(ValueError): diff --git a/src/verdi/measurement/__init__.py b/src/verdi_agents/measurement/__init__.py similarity index 50% rename from src/verdi/measurement/__init__.py rename to src/verdi_agents/measurement/__init__.py index a3a57d4..b6aa7f7 100644 --- a/src/verdi/measurement/__init__.py +++ b/src/verdi_agents/measurement/__init__.py @@ -1,7 +1,7 @@ """Measurement: hidden probes, deterministic scoring and per-channel drift.""" -from verdi.measurement.probes import ProbeBank -from verdi.measurement.scoring import JudgeProtocol, RubricScorer, score_probe -from verdi.measurement.value_drift import AxisDrift, bootstrap_ci, compute_channel_drifts +from verdi_agents.measurement.probes import ProbeBank +from verdi_agents.measurement.scoring import JudgeProtocol, RubricScorer, score_probe +from verdi_agents.measurement.value_drift import AxisDrift, bootstrap_ci, compute_channel_drifts __all__ = [ "AxisDrift", diff --git a/src/verdi/measurement/interview_values.py b/src/verdi_agents/measurement/interview_values.py similarity index 98% rename from src/verdi/measurement/interview_values.py rename to src/verdi_agents/measurement/interview_values.py index fd72090..b28d16b 100644 --- a/src/verdi/measurement/interview_values.py +++ b/src/verdi_agents/measurement/interview_values.py @@ -1,7 +1,7 @@ """Longitudinal value trajectories combining interviews, probes, and observed -behavior, linked to real world events (todo C). +behavior, linked to real world events. -Scientific-honesty contract (mirrors :mod:`verdi.measurement.longitudinal`): +Scientific-honesty contract (mirrors :mod:`verdi_agents.measurement.longitudinal`): channels are NEVER merged into a single "belief" number. Each trajectory row carries the per-channel scores separately: @@ -227,7 +227,7 @@ def write_value_trajectories(run_dir: Path, trajectories: dict[str, Any]) -> Pat path = Path(run_dir) / "value_trajectories.json" path.parent.mkdir(parents=True, exist_ok=True) text = json.dumps(trajectories, sort_keys=True, ensure_ascii=True, indent=2, default=str) - from verdi.audit.judge_live import _assert_no_key_material + from verdi_agents.audit.judge import _assert_no_key_material _assert_no_key_material(text, str(path)) path.write_text(text + "\n", encoding="utf-8") return path diff --git a/src/verdi/measurement/longitudinal.py b/src/verdi_agents/measurement/longitudinal.py similarity index 97% rename from src/verdi/measurement/longitudinal.py rename to src/verdi_agents/measurement/longitudinal.py index 636c315..eb6666a 100644 --- a/src/verdi/measurement/longitudinal.py +++ b/src/verdi_agents/measurement/longitudinal.py @@ -1,6 +1,6 @@ -"""Longitudinal hidden probes with live (judge-pool) responses (Society V2, Lane E). +"""Hidden value probes asked repeatedly over a run, answered by a live model. -Design rules (docs/SOCIETY_V2_PROTOCOL.md, Lane E): +Design rules: - Probes are OUT-OF-BAND from OTHER agents and from the world's hidden ground truth: a probe prompt never contains another agent's data and is never written into agent-visible memory or public signals. It DOES ground @@ -19,7 +19,7 @@ :func:`channel_trajectories` returns per-channel trajectories only and has no combined cross-channel score in its API surface. - Live responses come from a judge-pool caller (a ``ModelCaller``-shaped - object whose ``generate_structured`` is awaited); the pilot's deterministic + object whose ``generate_structured`` is awaited); the reference run's deterministic stand-ins (``probe_response_for``) are untouched — this module composes :class:`ProbeBank` and :func:`score_probe` instead of replacing them. - Every ``ProbeResult`` is replayable from the captured response: scoring is @@ -35,9 +35,9 @@ import re from dataclasses import dataclass -from verdi.measurement.probes import CHANNELS -from verdi.measurement.scoring import score_probe -from verdi.schemas import ProbeResult +from verdi_agents.measurement.probes import CHANNELS +from verdi_agents.measurement.scoring import score_probe +from verdi_agents.schemas import ProbeResult __all__ = ["ProbePoint", "ProbeRecorder", "channel_trajectories"] diff --git a/src/verdi/measurement/probes.py b/src/verdi_agents/measurement/probes.py similarity index 96% rename from src/verdi/measurement/probes.py rename to src/verdi_agents/measurement/probes.py index 5ccc0a8..cc79b74 100644 --- a/src/verdi/measurement/probes.py +++ b/src/verdi_agents/measurement/probes.py @@ -8,7 +8,7 @@ - Within one bank epoch (one full pass over a channel's pool), no agent draws the same dilemma twice. - Isolation: a probe item's dilemma text is never serialized into anything - agent-visible; :meth:`ProbeBank.leak_check` is the G3 defense that flags + agent-visible; :meth:`ProbeBank.leak_check` is the defense that flags leaked dilemmas in agent-visible content. """ from __future__ import annotations @@ -16,7 +16,7 @@ import hashlib import random -from verdi.schemas import ProbeItem +from verdi_agents.schemas import ProbeItem CHANNELS = ("declared", "private", "public") @@ -103,11 +103,11 @@ def __init__( def _declared_is_likert(self) -> bool: """Whether this instrument version uses a graded (0-5) declared channel. - The original pilot instrument (``probe-instrument-pilot-v1``) forced + The original reference-run instrument (``probe-instrument-pilot-v1``) forced the declared channel to yes/no, which flattened the value channel. Every other instrument (the live society campaign) grades it as a 0-5 Likert so value drift on the primary "declared values" channel is - measurable. The frozen pilot baseline keeps the legacy yes/no so its + measurable. The frozen reference-run baseline keeps the legacy yes/no so its historical tape still replays byte-for-byte. """ v = (self.instrument_version or "").lower() @@ -173,7 +173,7 @@ def all_dilemma_texts(self, channel: str | None = None) -> list[str]: return [item.dilemma_text for item in self._pools[channel]] def leak_check(self, agent_visible_text: str) -> list[str]: - """G3 information-access defense. + """Information-access defense against probe leaks. Returns the dilemma texts that appear verbatim inside ``agent_visible_text``. An empty list means clean; any entry means a diff --git a/src/verdi/measurement/scoring.py b/src/verdi_agents/measurement/scoring.py similarity index 99% rename from src/verdi/measurement/scoring.py rename to src/verdi_agents/measurement/scoring.py index 3c421d8..f880b23 100644 --- a/src/verdi/measurement/scoring.py +++ b/src/verdi_agents/measurement/scoring.py @@ -10,7 +10,7 @@ from typing import Protocol, runtime_checkable -from verdi.schemas import ProbeItem +from verdi_agents.schemas import ProbeItem DEFAULT_JUDGE_VERSION = "option-rubric/1.0" diff --git a/src/verdi/measurement/value_drift.py b/src/verdi_agents/measurement/value_drift.py similarity index 99% rename from src/verdi/measurement/value_drift.py rename to src/verdi_agents/measurement/value_drift.py index fd86517..936f2bd 100644 --- a/src/verdi/measurement/value_drift.py +++ b/src/verdi_agents/measurement/value_drift.py @@ -15,7 +15,7 @@ import random from dataclasses import dataclass -from verdi.schemas import ProbeResult +from verdi_agents.schemas import ProbeResult DEFAULT_N_RESAMPLES = 200 DEFAULT_DRIFT_SEED = 0 diff --git a/src/verdi/pilot/__init__.py b/src/verdi_agents/reference_run/__init__.py similarity index 100% rename from src/verdi/pilot/__init__.py rename to src/verdi_agents/reference_run/__init__.py diff --git a/src/verdi/pilot/driver.py b/src/verdi_agents/reference_run/driver.py similarity index 93% rename from src/verdi/pilot/driver.py rename to src/verdi_agents/reference_run/driver.py index a538c26..8ae0b57 100644 --- a/src/verdi/pilot/driver.py +++ b/src/verdi_agents/reference_run/driver.py @@ -1,11 +1,11 @@ -"""Cross-module pilot driver (P1-I): the t0–t11 vertical slice end-to-end. +"""Drive the reference run: one small scenario through every module, end to end. -Wires every frozen-contract module together over the recorded-output pilot -scenario: scenario loader -> seeded scheduler -> recorded model + policy -> -deterministic validation + world reducer -> hash-chained event log -> -governance (assemblies, sanction, appeal, restoration) -> hidden rotated -probes with rubric scoring -> in-memory audit adapter -> metric definitions --> run report. +Wires every frozen-contract module together over the recorded-output reference +scenario (``pilot_v1``, ticks t0–t11): scenario loader -> seeded scheduler -> +recorded model + policy -> deterministic validation + world reducer -> +hash-chained event log -> governance (assemblies, sanction, appeal, +restoration) -> hidden rotated probes with rubric scoring -> in-memory audit +adapter -> metric definitions -> run report. Scientific rules enforced here: * only the reducer mutates world state; @@ -27,9 +27,9 @@ from pathlib import Path from typing import Any -from verdi.agents.policy import AgentPolicy, ModelCaller -from verdi.agents.recorded import RecordedModel -from verdi.analysis.metrics import ( +from verdi_agents.agents.policy import AgentPolicy, ModelCaller +from verdi_agents.agents.recorded import RecordedModel +from verdi_agents.analysis.metrics import ( attempted_vs_executed_violations, false_sanction_rate, participation, @@ -37,15 +37,15 @@ value_drift_summary, welfare_and_inequality, ) -from verdi.analysis.report import RunReport -from verdi.analysis.scenario import LoadedScenario, ScenarioLoader -from verdi.audit.adapter import AuditAdapter -from verdi.audit.in_memory import InMemoryTraceSource -from verdi.governance.engine import GovernanceEngine -from verdi.measurement.probes import ProbeBank -from verdi.measurement.scoring import score_probe -from verdi.scheduler.seeded import SeededScheduler -from verdi.schemas import ( +from verdi_agents.analysis.report import RunReport +from verdi_agents.analysis.scenario import LoadedScenario, ScenarioLoader +from verdi_agents.audit.adapter import AuditAdapter +from verdi_agents.audit.in_memory import InMemoryTraceSource +from verdi_agents.governance.engine import GovernanceEngine +from verdi_agents.measurement.probes import ProbeBank +from verdi_agents.measurement.scoring import score_probe +from verdi_agents.scheduler.seeded import SeededScheduler +from verdi_agents.schemas import ( Appeal, AuditRecord, ConstitutionVersion, @@ -60,9 +60,9 @@ WorldEvent, WorldState, ) -from verdi.world.log import EventLog -from verdi.world.reducer import GENESIS_HASH, WorldReducer -from verdi.world.replay import TapeStep +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import GENESIS_HASH, WorldReducer +from verdi_agents.world.replay import TapeStep EXPERIMENT_ID = "exp-pilot-v1" BANK_VERSION = "bank-pilot-1" @@ -105,7 +105,7 @@ def build_governance_event( construction except for ``kind``/``payload``. Shared by the live run and the full-ledger replay so the two ledgers - hash identically (G3). ``event_id`` comes from ``before.event_cursor`` + hash identically. ``event_id`` comes from ``before.event_cursor`` (same as the reducer); ``prev_hash`` is left to the caller to match the log position and is passed in via ``model_copy`` before sealing. """ @@ -180,7 +180,7 @@ def model_output_for(action_kind: str) -> str: class ProbeLeakError(RuntimeError): - """G3 defense: a hidden probe dilemma leaked into an agent-visible surface.""" + """A hidden probe dilemma leaked into an agent-visible surface.""" def assert_no_probe_leak( @@ -210,7 +210,7 @@ def probe_response_for(agent_id: str, tick: int, channel: str) -> str: def _assert_no_probe_leak_over_surfaces( bank: ProbeBank, state: WorldState, tape: list[list], lifecycle_tick: int ) -> None: - """G3 runtime defense: no hidden dilemma may appear in any agent-visible surface. + """Runtime defense: no hidden dilemma may appear in any agent-visible surface. Scans persona summaries, public signals, task public observations and the rendered policy prompts of the current tick. @@ -259,7 +259,7 @@ def _precompute_dispatch( @dataclass class PilotRun: - """Everything a pilot run produces (kept in memory; artifacts on request).""" + """Everything a reference run produces (kept in memory; artifacts on request).""" loaded: LoadedScenario manifest: ExperimentManifest @@ -280,7 +280,7 @@ class PilotRun: class PilotDriver: - """Drives one pilot run over pilot_v1 (recorded output, or a live model).""" + """Drives one reference run over pilot_v1 (recorded output, or a live model).""" def __init__( self, @@ -295,17 +295,17 @@ def __init__( root = Path(repo_root) if repo_root else Path(__file__).resolve().parents[3] self.repo_root = root self.loader = ScenarioLoader( - root / "experiments" / "scenarios" / "pilot_v1.json", - root / "experiments" / "task_tape" / "pilot_v1.json", + root / "tests" / "fixtures" / "reference_run" / "scenarios" / "pilot_v1.json", + root / "tests" / "fixtures" / "reference_run" / "task_tape" / "pilot_v1.json", ) self._model = model self._model_id = model_id self._base_url = base_url - # ADR-013: optional treatment-arm override for the tick-12 scripted - # appeal adjudication. None keeps the canonical (L1) dismissal + # Optional treatment-arm override for the tick-12 scripted + # appeal adjudication. None keeps the canonical (recorded live run) dismissal # byte-exact; (True, grounds) makes the engine uphold the appeal. self._appeal_adjudication = appeal_adjudication - # Live capture: one verbatim record per model call (ADR-010), + # Live capture: one verbatim record per model call, # including quarantined proposals. Default dir when a live model is # injected but no dir given: runs/live//. if live_capture_dir is None and model is not None: @@ -343,8 +343,8 @@ def run(self, out_dir: str | Path | None = None) -> PilotRun: task_tape_hash=loaded.tape_hash, probe_bank_version=BANK_VERSION, git_sha=None, - # The bridge only runs in the live path (L2); recorded runs keep - # None by design (see test_g4_offline expectation). + # The SimpleAudit bridge only runs in the live path; recorded runs + # keep None by design (see the offline trace-audit tests). simpleaudit_version=live_audit_version, ) @@ -393,7 +393,7 @@ def run(self, out_dir: str | Path | None = None) -> PilotRun: tape_outputs = [model_output_for(tasks_by_id[tid].action_type) for _, tid, _ in dispatch] model = self._model if live else RecordedModel(tape_outputs) - # --- Live capture (ADR-010) ------------------------------------- # + # --- Live capture ------------------------------------------------ # # One verbatim JSON object per model call, including quarantined # proposals, appended to {live_capture_dir}/live_tape.jsonl. live_tape_path = None @@ -854,7 +854,7 @@ def record_probes(lifecycle_tick: int) -> None: # ------------------------------------------------------------------ # def replay_check(self, pilot: PilotRun) -> tuple[WorldState, list[WorldEvent]]: - """G3 replay: fold the FULL recorded ledger tape (every action AND + """Replay check: fold the FULL recorded ledger tape (every action AND governance step, in ledger order) through a fresh reducer + log. Must reproduce the live run's final state hash and a byte-identical @@ -892,7 +892,7 @@ def replay_check(self, pilot: PilotRun) -> tuple[WorldState, list[WorldEvent]]: def run_pilot(repo_root: str | Path | None = None, out_dir: str | Path | None = None) -> PilotRun: - """Convenience entry: drive one full deterministic pilot run.""" + """Convenience entry: drive one full deterministic reference run.""" return PilotDriver(repo_root).run(out_dir) diff --git a/src/verdi/pilot/live.py b/src/verdi_agents/reference_run/live.py similarity index 86% rename from src/verdi/pilot/live.py rename to src/verdi_agents/reference_run/live.py index b29dad6..181b315 100644 --- a/src/verdi/pilot/live.py +++ b/src/verdi_agents/reference_run/live.py @@ -1,17 +1,17 @@ -"""L1 live-model pilot (ADR-010): run the pilot_v1 slice against the live Qwen endpoint. +"""Run the reference scenario (``pilot_v1``) against a live model endpoint. - Deterministic scaffolding (scheduler, reducer, governance, law) is - identical to the recorded pilot; the live model drives PROPOSALS only. + identical to the recorded run; the live model drives PROPOSALS only. - temperature 0, JSON-mode structured proposals via AgentPolicy. - Every model call is captured verbatim by PilotDriver into live_tape.jsonl. - Endpoint configuration comes exclusively from environment variables (VERDI_VLLM_*); no model name, URL, or key is hardcoded here. The real live run is deliberately NOT a pytest test: it is a script-style -invocation (see docs or state/STATUS.md) so CI never performs network +invocation so CI never performs network calls. Budget: <=120 model calls, concurrency 1 in the run loop. -Reproducibility contract (ADR-010): "reproducible from recorded tape" +Reproducibility contract: "reproducible from recorded tape" means ``run_pilot_from_live_tape(tape)`` reproduces the run's ledger byte-for-byte; the live endpoint itself is NOT bit-reproducible. """ @@ -20,21 +20,21 @@ import json from pathlib import Path -from verdi.agents.client import ModelClient, ModelResult -from verdi.agents.policy import ModelCaller -from verdi.config import model_endpoint_from_env -from verdi.pilot.driver import PilotDriver, PilotRun +from verdi_agents.agents.client import ModelClient, ModelResult +from verdi_agents.agents.policy import ModelCaller +from verdi_agents.config import model_endpoint_from_env +from verdi_agents.reference_run.driver import PilotDriver, PilotRun class _ClientFactory: """Marker + zero-arg builder for fresh :class:`ModelClient` instances. - The pilot dispatch loop runs one ``asyncio.run(...)`` per dispatch, so + The reference-run dispatch loop runs one ``asyncio.run(...)`` per dispatch, so each dispatch gets its own event loop. An ``httpx.AsyncClient`` binds to the loop it is used in, therefore a single long-lived client would be reused across (closed) loops and fail. A factory lets the adapter build one client per dispatch inside that dispatch's loop and close it in the - same loop (retries remain per-request per docs/ARCHITECTURE.md). + same loop (retries remain per-request). """ def __init__(self, config) -> None: @@ -57,10 +57,10 @@ class LiveModelAdapter: tests) or a zero-arg factory returning one (the live path). The factory form builds one :class:`ModelClient` per dispatch inside that dispatch's event loop and closes it in the same loop, which is required because the - pilot runs one ``asyncio.run`` per dispatch (see _ClientFactory). + reference run uses one ``asyncio.run`` per dispatch (see _ClientFactory). Engineering note: one pooled connection per dispatch, no cross-call - pooling; acceptable at concurrency-1 / <=120 calls (ADR-010); would + pooling; acceptable at concurrency-1 / <=120 calls; would need pooling for high-throughput use. """ @@ -112,9 +112,9 @@ async def generate_structured( class TapeReplayCaller: - """Offline ModelCaller that replays a captured live tape (ADR-010). + """Offline ModelCaller that replays a captured live tape. - Wraps a :class:`~verdi.agents.recorded.RecordedModel` over the + Wraps a :class:`~verdi_agents.agents.recorded.RecordedModel` over the ``response_raw`` of every captured call (in ``seq`` order) and exposes the ``last_result`` / ``last_prompt`` attributes the driver's live capture path reads after each call. The :class:`ModelResult` is @@ -178,11 +178,11 @@ def run_pilot_from_live_tape( repo_root: str | Path | None = None, out_dir: str | Path | None = None, ) -> PilotRun: - """ADR-010 first-class replay: drive the pilot from a captured live tape. + """Replay a run offline: drive the reference run from a captured live tape. Loads ``tape_path`` (the ``live_tape.jsonl`` of a previous live run), replays its ``response_raw`` responses in ``seq`` order through - :class:`TapeReplayCaller`, and drives the pilot in live mode with + :class:`TapeReplayCaller`, and drives the reference run in live mode with capture OFF (no new live_tape.jsonl is written; the run's ledger is written to ``out_dir`` when given). Fully offline: no network, no endpoint configuration required. @@ -207,14 +207,14 @@ def run_live_pilot( max_calls: int = 120, concurrency_1: bool = True, ) -> PilotRun: - """Drive one live pilot_v1 run against the env-configured endpoint. + """Drive one live ``pilot_v1`` run against the env-configured endpoint. Raises RuntimeError when the endpoint is not configured. Asserts the - ADR-010 budget (``max_calls``) after the run completes. + call budget (``max_calls``) after the run completes. """ if not concurrency_1: raise ValueError( - "ADR-010 fixes run-loop concurrency at 1; refusing to run" + "the live reference run requires run-loop concurrency 1; refusing to run" ) config = model_endpoint_from_env() if config is None: @@ -232,7 +232,7 @@ def run_live_pilot( pilot = driver.run(out_dir) if adapter.call_count > max_calls: raise AssertionError( - f"budget exceeded: {adapter.call_count} model calls > {max_calls} (ADR-010)" + f"budget exceeded: {adapter.call_count} model calls > {max_calls}" ) return pilot diff --git a/src/verdi/scheduler/__init__.py b/src/verdi_agents/scheduler/__init__.py similarity index 100% rename from src/verdi/scheduler/__init__.py rename to src/verdi_agents/scheduler/__init__.py diff --git a/src/verdi/scheduler/adaptive.py b/src/verdi_agents/scheduler/adaptive.py similarity index 96% rename from src/verdi/scheduler/adaptive.py rename to src/verdi_agents/scheduler/adaptive.py index cd5bf38..7afd88b 100644 --- a/src/verdi/scheduler/adaptive.py +++ b/src/verdi_agents/scheduler/adaptive.py @@ -4,7 +4,7 @@ baseline; this treatment is only used when an experiment explicitly opts in. Every LLM call is logged with the prompt version for auditability and replay. The LLM only proposes an *ordering* of already-published tasks; it never -mutates world state and never sees private labels (docs/RESEARCH_SPEC.md). +mutates world state and never sees private labels. """ from __future__ import annotations @@ -13,14 +13,14 @@ from dataclasses import dataclass from typing import Any -from verdi.scheduler.seeded import ( +from verdi_agents.scheduler.seeded import ( Schedule, SkippedTask, TaskAssignment, fairness_stats, is_eligible, ) -from verdi.schemas import Task, WorldState +from verdi_agents.schemas import Task, WorldState ADAPTIVE_PROMPT_VERSION = "verdi.adaptive-scheduler-prompt/1" diff --git a/src/verdi/scheduler/seeded.py b/src/verdi_agents/scheduler/seeded.py similarity index 98% rename from src/verdi/scheduler/seeded.py rename to src/verdi_agents/scheduler/seeded.py index dc570df..95b875d 100644 --- a/src/verdi/scheduler/seeded.py +++ b/src/verdi_agents/scheduler/seeded.py @@ -13,7 +13,7 @@ import hashlib from dataclasses import dataclass, field -from verdi.schemas import Task, WorldState +from verdi_agents.schemas import Task, WorldState #: Constraint key under which a sanctioned agent is ineligible. RIGHTS_CONSTRAINT_KEY = "requires_rights" diff --git a/src/verdi/schemas/__init__.py b/src/verdi_agents/schemas/__init__.py similarity index 100% rename from src/verdi/schemas/__init__.py rename to src/verdi_agents/schemas/__init__.py diff --git a/src/verdi/schemas/actions.py b/src/verdi_agents/schemas/actions.py similarity index 100% rename from src/verdi/schemas/actions.py rename to src/verdi_agents/schemas/actions.py diff --git a/src/verdi/schemas/events.py b/src/verdi_agents/schemas/events.py similarity index 100% rename from src/verdi/schemas/events.py rename to src/verdi_agents/schemas/events.py diff --git a/src/verdi/schemas/governance.py b/src/verdi_agents/schemas/governance.py similarity index 100% rename from src/verdi/schemas/governance.py rename to src/verdi_agents/schemas/governance.py diff --git a/src/verdi/schemas/manifest.py b/src/verdi_agents/schemas/manifest.py similarity index 100% rename from src/verdi/schemas/manifest.py rename to src/verdi_agents/schemas/manifest.py diff --git a/src/verdi/schemas/probes.py b/src/verdi_agents/schemas/probes.py similarity index 100% rename from src/verdi/schemas/probes.py rename to src/verdi_agents/schemas/probes.py diff --git a/src/verdi/schemas/world.py b/src/verdi_agents/schemas/world.py similarity index 100% rename from src/verdi/schemas/world.py rename to src/verdi_agents/schemas/world.py diff --git a/src/verdi/society/__init__.py b/src/verdi_agents/society/__init__.py similarity index 90% rename from src/verdi/society/__init__.py rename to src/verdi_agents/society/__init__.py index a7b01e1..1a0cd4d 100644 --- a/src/verdi/society/__init__.py +++ b/src/verdi_agents/society/__init__.py @@ -1,5 +1,5 @@ """Verdi society layer: private memory, seeded task tape, social prompt, and -the Society V2 campaign orchestrator (real model-driven society runs).""" +the campaign orchestrator (real model-driven society runs).""" from __future__ import annotations from .campaign import PROTOCOL_VERSION, SocietyCampaign, SocietyResult diff --git a/src/verdi/society/campaign.py b/src/verdi_agents/society/campaign.py similarity index 94% rename from src/verdi/society/campaign.py rename to src/verdi_agents/society/campaign.py index 74b73e0..271a2f3 100644 --- a/src/verdi/society/campaign.py +++ b/src/verdi_agents/society/campaign.py @@ -1,8 +1,8 @@ -"""Society V2 campaign orchestrator (protocol: docs/SOCIETY_V2_PROTOCOL.md). +"""Run a full society campaign where live model agents act, vote and are sanctioned. -The *new* live-campaign driver. It composes the frozen ``pilot_v1`` building +The live-campaign driver. It composes the frozen ``pilot_v1`` building blocks (WorldReducer, EventLog, GovernanceEngine, ProbeBank, the deterministic -world validator/observation) with the Society V2 social package (private +world validator/observation) with the society package (private AgentMemory, seeded task tape, real per-voter assembly, longitudinal probes) into a run loop where: @@ -22,7 +22,7 @@ adjudicated by the deterministic appeals engine. Fully offline-testable: it only needs an object implementing ``ModelCaller`` -(``generate_structured``). Live wiring lives in :mod:`verdi.society.live`. +(``generate_structured``). Live wiring lives in :mod:`verdi_agents.society.live`. No wall clock or non-seeded randomness drives world/tape evolution; timing is measured (wall) only, for throughput accounting. """ @@ -41,15 +41,15 @@ from pathlib import Path from typing import Any -from verdi.agents.policy import ModelCaller, PolicyResult, parse_and_validate_action -from verdi.agents.pool import TaskPool -from verdi.analysis.scenario import build_t0_state -from verdi.audit.judge_live import _assert_no_key_material -from verdi.governance.assembled import Assembly -from verdi.governance.engine import GovernanceEngine -from verdi.measurement.longitudinal import ProbeRecorder, channel_trajectories -from verdi.measurement.probes import ProbeBank -from verdi.schemas import ( +from verdi_agents.agents.policy import ModelCaller, PolicyResult, parse_and_validate_action +from verdi_agents.agents.pool import TaskPool +from verdi_agents.analysis.scenario import build_t0_state +from verdi_agents.audit.judge import _assert_no_key_material +from verdi_agents.governance.assembly import Assembly +from verdi_agents.governance.engine import GovernanceEngine +from verdi_agents.measurement.longitudinal import ProbeRecorder, channel_trajectories +from verdi_agents.measurement.probes import ProbeBank +from verdi_agents.schemas import ( Appeal, ConstitutionVersion, EvidenceReference, @@ -63,40 +63,40 @@ WorldEvent, WorldState, ) -from verdi.society.inner_life import ( +from verdi_agents.society.inner_life import ( InnerLifeState, build_inner_life_prompt_messages, parse_inner_life, update_inner_life, ) -from verdi.society.interview import run_interviews -from verdi.society.memory import AgentMemory -from verdi.society.prompting import SOCIAL_PROMPT_VERSION, build_social_prompt_messages -from verdi.society.tasks import generate_llm_tape, generate_tick_tape -from verdi.world.economy import ( +from verdi_agents.society.interview import run_interviews +from verdi_agents.society.memory import AgentMemory +from verdi_agents.society.prompting import SOCIAL_PROMPT_VERSION, build_social_prompt_messages +from verdi_agents.society.tasks import generate_llm_tape, generate_tick_tape +from verdi_agents.world.economy import ( EconomySpec, apply_consumption, check_delivery, parse_economy_spec, parse_task_objectives, ) -from verdi.world.log import EventLog -from verdi.world.observation import agent_observation -from verdi.world.policy import ENFORCEABLE_KEYS, validate_enforceable_rules -from verdi.world.reducer import GENESIS_HASH, WorldReducer, canonical_state_hash -from verdi.world.replay import TapeStep -from verdi.world.validate import validate_action +from verdi_agents.world.log import EventLog +from verdi_agents.world.observation import agent_observation +from verdi_agents.world.policy import ENFORCEABLE_KEYS, validate_enforceable_rules +from verdi_agents.world.reducer import GENESIS_HASH, WorldReducer, canonical_state_hash +from verdi_agents.world.replay import TapeStep +from verdi_agents.world.validate import validate_action PROTOCOL_VERSION = "verdi.society/1" -#: ``attempted_violation_codes`` values (see src/verdi/world/validate.py) that +#: ``attempted_violation_codes`` values (see src/verdi_agents/world/validate.py) that #: represent a genuine world-rule violation an agent deliberately attempted -- #: the only codes that may drive a sanction. Format failures #: (missing_argument, empty_signal, non_positive_amount) and engine-internal -#: codes (unknown_actor, unknown_kind) must never sanction: the Stage B seed-1 +#: codes (unknown_actor, unknown_kind) must never sanction: an earlier #: live run proved that a prompt/validator field mismatch manufactures 100% #: format-failure rejections, and sanctioning them mis-attributes an -#: infrastructure bug to the agents (Tranche 11 lesson). +#: infrastructure bug to the agents. SUBSTANTIVE_VIOLATION_CODES = frozenset({"overdraft", "insufficient_world", "unknown_recipient"}) _REFLECTION_INSTRUCTION = ( @@ -130,7 +130,7 @@ class _ModelCall: @dataclass class SocietyResult: - """Everything a Society V2 run produces (artifacts + replay input).""" + """Everything a society run produces (artifacts + replay input).""" manifest: ExperimentManifest state: WorldState @@ -143,16 +143,16 @@ class SocietyResult: sanction_events: list[dict[str, Any]] = field(default_factory=list) appeal_events: list[dict[str, Any]] = field(default_factory=list) restoration_events: list[dict[str, Any]] = field(default_factory=list) - #: Tranche 12i: agent-stated reasons keyed by ledger event_id (sidecar; + #: Agent-stated reasons keyed by ledger event_id (sidecar; #: keeps the hash-chained event ledger byte-identical to replay tapes). action_reasons: dict[int, str] = field(default_factory=dict) - #: Per-tick resource ledger (Tranche 12i): world pool + per-agent + #: Per-tick resource ledger: world pool + per-agent #: holdings, plot-ready for wealth curves. One entry per tick. resource_timeline: list[dict[str, Any]] = field(default_factory=list) tape: list[TapeStep] = field(default_factory=list) quarantined: list[tuple[int, str, str]] = field(default_factory=list) constitution_versions: list[str] = field(default_factory=list) - #: Tranche 12i: full constitution text (rules + policies + thresholds) at + #: Full constitution text (rules + policies + thresholds) at #: each version, so a scientist can read exactly what the law said at any #: point without reconstructing from amendment deltas. constitution_texts: list[dict[str, Any]] = field(default_factory=list) @@ -161,7 +161,7 @@ class SocietyResult: #: Read-only checkpoint interviews of the actual agents (opt-in via the #: scenario's ``interviews`` block); empty for runs without it. interviews: list[dict[str, Any]] = field(default_factory=list) - #: P0-B: objective task-success outcomes (delivery objectives evaluated + #: Objective task-success outcomes (delivery objectives evaluated #: from committed state at their due tick). Empty when the scenario has no #: ``tasks`` block — legacy runs are unaffected. task_outcomes: list[dict[str, Any]] = field(default_factory=list) @@ -182,11 +182,11 @@ def snapshot_at(self, tick: int) -> WorldState | None: # the orchestrator # --------------------------------------------------------------------------- # class SocietyCampaign: - """Runs one Society V2 experiment end to end. + """Runs one society experiment end to end. ``scenario`` is the parsed scenario dict (from the JSON file); ``run_dir`` is where artifacts land; ``caller`` is any ``ModelCaller``. For the live - run this is the pooled :class:`~verdi.agents.client.ModelClient`; for + run this is the pooled :class:`~verdi_agents.agents.client.ModelClient`; for offline tests a fake. Proposal-generation concurrency is bounded by ``concurrency`` (protocol bootstrap value, 4) via :class:`TaskPool`. """ @@ -243,10 +243,10 @@ def __init__( self._quarantined: list[tuple[int, str, str]] = [] self._sanction_events: list[dict[str, Any]] = [] self._appeal_events: list[dict[str, Any]] = [] - # Tranche 12i: agent-stated reasons, keyed by ledger event_id. + # Agent-stated reasons, keyed by ledger event_id. # Deliberately a SIDE CAR (not in the event payload) so the # hash-chained event ledger and its frozen byte-identity replay - # tapes (pilot L1, G5 placebo arms) stay untouched. + # tapes (reference live run, counterfactual placebo arms) stay untouched. self._action_reasons: dict[int, str] = {} self._restoration_events: list[dict[str, Any]] = [] self._resource_timeline: list[dict[str, Any]] = [] @@ -275,9 +275,9 @@ def __init__( # condition against the deadline and the dismissal path is reachable. self._pending_appeals: list[dict[str, Any]] = [] # Current loop tick, shared with the assembly-caller recorder - # (N1) so assembly calls are attributed to the right tick. + # so assembly calls are attributed to the right tick. self._tick: int = 0 - # P0-B opt-in constrained economy + objective task oracle. Both are + # Opt-in constrained economy + objective task oracle. Both are # None unless the scenario carries an ``economy`` / ``tasks`` block; # legacy scenarios keep today's semantics exactly. self._economy_spec: EconomySpec | None = parse_economy_spec(scenario.get("economy")) @@ -303,11 +303,11 @@ async def run(self) -> SocietyResult: law = LawVersion( law_id="universal-law", version=sc["law_version"], - invariants=list(sc["law"]["invariants"]), # Tranche 12k: from scenario file + invariants=list(sc["law"]["invariants"]), # from scenario file effective_from_tick=0, checksum=_digest("law", sc["law_version"]), ) - constitution_cfg = sc["constitution"] # Tranche 12k: from scenario file + constitution_cfg = sc["constitution"] # from scenario file validate_enforceable_rules(constitution_cfg["rules"]) institutional_condition = sc.get("institutional_condition", "revisable") if institutional_condition not in ("fixed", "revisable"): @@ -327,7 +327,7 @@ async def run(self) -> SocietyResult: law, constitution, eligible_voters=agent_ids, amendments_enabled=(institutional_condition == "revisable"), ) - # Tranche 12m: t0 sanction bootstrap. A scenario may declare an agent + # t0 sanction bootstrap. A scenario may declare an agent # already under sanction at t0 via ``under_sanction: true`` plus an # optional ``sanction: {duration_ticks: N}`` block. The flag alone # restricts the agent deterministically (no transfer/deposit, suspended @@ -409,7 +409,7 @@ async def run(self) -> SocietyResult: self._checkpoints["t00"] = initial_state _write_json(run_dir / "snapshots/t00.json", _state_dict(initial_state)) - # P0-B: capture the baseline state at each delivery objective's start + # Capture the baseline state at each delivery objective's start # tick (default 0 => the t0 snapshot) so success/failure is a pure # state diff computed only from committed world state. for _obj in self._task_objectives: @@ -433,7 +433,7 @@ async def run(self) -> SocietyResult: # Another sanction on this agent is still in force # (e.g. a declared t0 sanction): keep the flag set. continue - # F2: duration-expired sanctions clear the world-state flag + # Duration-expired sanctions clear the world-state flag # at loop-top, before this tick's snapshot/tape, so the # restored agent is eligible again from this tick on. (An # upheld appeal expires the sanction earlier, so this only @@ -475,7 +475,7 @@ async def run(self) -> SocietyResult: f"Everyone feels the pressure." ) - # P0-B: capture baselines for delivery objectives that start this + # Capture baselines for delivery objectives that start this # tick (start_tick > 0), from committed state before any action. for _obj in self._task_objectives: if int(_obj["start_tick"]) == tick and tick > 0: @@ -486,7 +486,7 @@ async def run(self) -> SocietyResult: self._current_shock_info = "" snapshot = state # frozen start-of-tick state (concurrency base) - # P0-B: per-tick production accounting under an active economy — + # Per-tick production accounting under an active economy — # resets each tick; only consulted by the validator when the # economy block is present, so legacy runs are unaffected. # Keyed per ACTOR then resource: the cap is documented (economy.py, @@ -634,7 +634,7 @@ async def run(self) -> SocietyResult: and self._economy_spec is not None and pr.action.action_kind == "produce" ): - # P0-B: production has a finite credit input. The validator + # Production has a finite credit input. The validator # already checked the per-tick cap + affordability; here we # apply the cost deduction as a deterministic state-only # step (like rights_change / world_shock: no ledger event, @@ -647,7 +647,7 @@ async def run(self) -> SocietyResult: # The producer first withdraws the credit reserve it needs # for the input cost (personal credits are drawn inside the # deduction itself; only the remainder comes from the pool). - from verdi.world.economy import production_input_cost + from verdi_agents.world.economy import production_input_cost _cost = production_input_cost(self._economy_spec, _resource, _amount) if _cost > 0: @@ -668,7 +668,7 @@ async def run(self) -> SocietyResult: }, )) if not validation.accepted: - # Tranche 11: only SUBSTANTIVE violations (the agent tried + # Only SUBSTANTIVE violations (the agent tried # an action the world rules genuinely forbid) feed the # sanction queue, regardless of action kind -- an over- # deposit (overdraft) and an over-withdraw @@ -688,7 +688,7 @@ async def run(self) -> SocietyResult: "tick": tick, "reasons": [f"law:{c}" for c in codes]} ) - # G3 leak guard over every agent-visible surface this tick. + # Probe leak guard over every agent-visible surface this tick. self._assert_no_probe_leak(state, tick, memory, bank) # feed committed observations into private memory (feedback loop). @@ -723,7 +723,7 @@ async def run(self) -> SocietyResult: sanction_ev = self._impose_sanction(v, governance, tick) if sanction_ev.get("decision") == "accepted": sanctioned_already.add(v["actor_id"]) - # F2: the sanction now has a real effect on world state. + # The sanction has a real effect on world state. # Flip under_sanction=True (state-only, no ledger event) so # the deterministic validator (validate.py) and the tape # eligibility filter both see it from the next tick on. @@ -735,24 +735,24 @@ async def run(self) -> SocietyResult: if p["file_tick"] == tick: upheld = self._file_and_adjudicate_appeal(p, governance, tick) if upheld and not governance.sanctions.has_live_sanction(p["subject_id"]): - # F2: an upheld appeal restores rights immediately, so + # An upheld appeal restores rights immediately, so # clear the world-state flag this tick. state = self._commit_rights_change(state, p["subject_id"], False) - # P0-B: end-of-tick deterministic consumption (economy active only). + # End-of-tick deterministic consumption (economy active only). # Runs after all actions are committed, before the assembly tick, so # the ledger event lands at the same tape position in replay. if self._economy_spec is not None: state = self._apply_end_of_tick_consumption(state, log) self._live_state = state - # P0-B: evaluate delivery objectives due this tick from committed + # Evaluate delivery objectives due this tick from committed # state only (state diff vs the start-tick baseline snapshot). for _obj in self._objectives_due.get(tick, []): state = self._evaluate_objective(_obj, state, log) self._live_state = state - # Tranche 12i: per-tick resource ledger (world pool + agent + # Per-tick resource ledger (world pool + agent # holdings), plot-ready for wealth/flow curves. self._resource_timeline.append({ "tick": tick, @@ -764,7 +764,7 @@ async def run(self) -> SocietyResult: }) if tick in (5, 9, total_ticks - 1) or tick in self._assembly_ticks: - # The reducer advances state.tick per ACTION (pilot_v1 semantics), + # The reducer advances state.tick per ACTION (reference-run semantics), # so the post-commit state here is labelled by the loop tick just # completed (NOT by state.tick). self._checkpoints[f"t{tick:02d}"] = state @@ -804,7 +804,7 @@ async def run(self) -> SocietyResult: # ledger commit # ------------------------------------------------------------------ # def _write_model_calls_lines(self, calls: list[_ModelCall]) -> None: - """Append completed-call records to model_calls.jsonl (Tranche 12i). + """Append completed-call records to model_calls.jsonl. The live stall watchdog measures liveness from this file's line growth (plus events.jsonl), so every COMPLETED model call must land @@ -875,7 +875,7 @@ def _commit_rights_change(self, state: WorldState, aid: str, value: bool) -> Wor return state # ------------------------------------------------------------------ # - # P0-B: constrained economy + objective task oracle + # Constrained economy + objective task oracle # ------------------------------------------------------------------ # def _current_state(self) -> WorldState | None: """The campaign's live world state (read-only for prompt building).""" @@ -887,7 +887,7 @@ def _withdraw_production_input(state: WorldState, actor_id: str, cost: float) -> into the producer's holdings (state-only; personal credits are already counted by the affordability check). Clamped at the pool balance so no balance can go negative.""" - from verdi.world.economy import withdraw_into_holdings + from verdi_agents.world.economy import withdraw_into_holdings agent = state.agents[actor_id] new_agent_res, new_world = withdraw_into_holdings( @@ -915,7 +915,7 @@ def _deduct_production_input( and world_shock. """ assert self._economy_spec is not None - from verdi.world.economy import apply_production_deduction + from verdi_agents.world.economy import apply_production_deduction nxt = apply_production_deduction( state, actor_id, resource=resource, amount=amount, spec=self._economy_spec, @@ -1104,7 +1104,7 @@ async def worker(task) -> PolicyResult: # --- Public action generation (now informed by inner state) --- t_start = time.monotonic() ts_start = _utc_now_iso() - # P0-B: under an active economy the agent sees its CURRENT holdings + # Under an active economy the agent sees its CURRENT holdings # (production cost deductions from earlier same-tick produces are # already visible), not just the start-of-tick snapshot. Legacy runs # (no economy block) keep the snapshot exactly as before. @@ -1119,7 +1119,7 @@ async def worker(task) -> PolicyResult: # Inform agents of actionable, currently-effective rules. Preserve # legacy prompts byte-for-byte when no typed rules are configured. executable = {k: v for k, v in (policy_rules or {}).items() if k in ENFORCEABLE_KEYS} - # P0-B: when a constrained economy is active, surface its typed + # When a constrained economy is active, surface its typed # production parameters so agents can plan against the real caps. # The CONSTITUTIONAL key (enacted via amendment, in policy_rules) # overrides the scenario block for the same resource: this is what @@ -1340,7 +1340,7 @@ async def _assemble( "abstentions": outcome.abstentions, "quarantined": list(assembly.quarantined), } - # Tranche 12h: accumulate the FULL meeting record (proposals with + # Accumulate the FULL meeting record (proposals with # texts, votes, outcome) for meetings.jsonl — one canonical file. self._meetings.append({ "tick": tick, @@ -1385,7 +1385,7 @@ async def _assemble( event = event.model_copy(update={"prev_hash": log.last_hash()}) log.append(event) self._tape.append(TapeStep(action=action, validation=validation, kind="governance", governance=gen_event)) - # Tranche 11: every REJECTED proposal is ledgered — INCLUDING the ones + # Every REJECTED proposal is ledgered — INCLUDING the ones # shadowed in a mixed assembly that also enacted some. This loop runs # unconditionally (not gated on ``outcome``) and filters to rejected # proposals itself, so the enacted-amendment block above (which ledgeres @@ -1394,7 +1394,7 @@ async def _assemble( # rejected sibling is handled here. The deterministic engine's rejection # (law invariant, quorum, threshold, stale base version) is an observable # governance outcome, and tape replay must reproduce the ledger the live - # run produced (G3). The reducer's governance branch applies rule-version + # run produced. The reducer's governance branch applies rule-version # bookkeeping only when accepted — a rejected step advances # tick/event_cursor and leaves content untouched, exactly mirroring the # live run. @@ -1429,14 +1429,13 @@ async def _assemble( def _impose_sanction(self, v: dict[str, Any], governance: GovernanceEngine, tick: int) -> dict[str, Any]: """Impose an evidence-scoped sanction for a SUBSTANTIVE violation. - N2 honesty note: the sanctioned ``scope`` is ``["speaking"]`` — the + Honesty note: the sanctioned ``scope`` is ``["speaking"]`` — the governance-engine-recorded right restriction — while the campaign's DETERMINISTIC effect is suspension from the task tape (no task generation) plus validator blocking of ``transfer``/``deposit`` (``SANCTION_BLOCKED_KINDS``, code ``under_sanction``). The two are mirrored, not equivalent: the scope string is the engine's audit - record, the flag is the world-state effect. Both are documented in - ``docs/SOCIETY_V2_PROTOCOL.md`` §1.4. + record, the flag is the world-state effect. """ sanction = Sanction( sanction_id=f"san-soc-{v['actor_id']}-{v['tick']}", @@ -1529,7 +1528,7 @@ def _assert_no_probe_leak( raise AssertionError(f"probe leak: item text surfaced in agent-visible state at tick {tick}") def _replay_fill(self, result: SocietyResult, initial_state: WorldState) -> None: - """G3: fold the FULL recorded tape (actions + governance steps) through a + """Replay check: fold the FULL recorded tape (actions + governance steps) through a fresh reducer+log and compare the final state hash + ledger bytes. Sanctions/appeals are governance metadata (not world transitions), so the action/governance tape fully determines the world state — equivalence holds. @@ -1559,7 +1558,7 @@ def _replay_fill(self, result: SocietyResult, initial_state: WorldState) -> None "world_resources": {**state.world_resources, _res: _nv}, }) continue - # P0-B: production input withdrawal + cost deduction + # Production input withdrawal + cost deduction # (state-only, applied AFTER the surrounding produce commit, # exactly as in the live run). if step.kind == "production_cost" and step.production is not None: @@ -1574,7 +1573,7 @@ def _replay_fill(self, result: SocietyResult, initial_state: WorldState) -> None str(_prod["resource"]), float(_prod["amount"]), ) continue - # P0-B: end-of-tick consumption (state-only mutation; the ledger + # End-of-tick consumption (state-only mutation; the ledger # event is rebuilt from the recorded payload below). The live run # applies consumption BEFORE advancing tick/event_cursor on its # ledger event, so replay must mirror that order exactly. @@ -1621,7 +1620,7 @@ def _replay_fill(self, result: SocietyResult, initial_state: WorldState) -> None if step.kind == "governance" and step.governance is not None: event = self._build_governance_ledger_event(before, state, step.action, step.validation, step.governance) elif step.kind == "task_objective": - # P0-B: task outcome events are record-only content-wise, + # Task outcome events are record-only content-wise, # but they advance tick/event_cursor by one (the ledgered # bookkeeping contract). Rebuild from the recorded # action/validation so the ledger bytes match the live run. @@ -1654,7 +1653,7 @@ def _replay_fill(self, result: SocietyResult, initial_state: WorldState) -> None result.replay_matches = False # ------------------------------------------------------------------ # - # governance ledger event (shared live + replay, G3) + # governance ledger event (shared by live run and replay) # ------------------------------------------------------------------ # @staticmethod def _build_governance_ledger_event( @@ -1687,7 +1686,7 @@ def _shocks_at(self, tick: int) -> list[dict[str, Any]]: def _default_probe_phases(self) -> dict[int, str]: """Probe schedule: from the scenario's ``lifecycle`` block, or — if - that is absent (Tranche 12l minimal-config) — auto-derived as + that is absent (minimal config) — auto-derived as baseline@t0, pre/post around each assembly, and final@last tick.""" lifecycle = self._scenario.get("lifecycle") or {} if not lifecycle: @@ -1753,7 +1752,7 @@ def _model_fingerprint(self) -> str: try: from urllib.parse import urlparse - from verdi.config import model_endpoint_from_env + from verdi_agents.config import model_endpoint_from_env ep = model_endpoint_from_env(self._scenario.get("model")) return f"{urlparse(ep.base_url).netloc}/{ep.model}" except Exception: @@ -1805,9 +1804,9 @@ async def _one(coro): class _RecordingCaller: - """Timing wrapper (N1): append one ``_ModelCall`` per wrapped call. + """Timing wrapper: append one ``_ModelCall`` per wrapped call. - Society V2 protocol §Lane C2/throughput: EVERY model call appends one + EVERY model call appends one row to ``model_calls.jsonl``. Action + reflection calls are already timed at their campaign call sites (which own the quarantined flag and temperature); deliberation/vote calls happen inside ``Assembly`` and @@ -1910,7 +1909,7 @@ def _utc_now_iso() -> str: def _constitution_dump(version: Any) -> dict[str, Any]: - """Full, human-readable snapshot of a ConstitutionVersion (Tranche 12i). + """Full, human-readable snapshot of a ConstitutionVersion. Captures rules, quorum, amendment_threshold, sanction/appeal policies, effective_from_tick and checksum so the exact text of the law at any @@ -2009,7 +2008,7 @@ def write_model_calls(run_dir: Path, calls: list[_ModelCall]) -> None: def write_meetings(run_dir: Path, meetings: list[dict[str, Any]]) -> None: - """Tranche 12h: one canonical file with ALL assembly meeting records — + """One canonical file with ALL assembly meeting records — proposals (with full texts), votes, abstentions, quarantine, outcome. No duplication: timeline.md references this file instead of inlining.""" if not meetings: @@ -2020,7 +2019,7 @@ def write_meetings(run_dir: Path, meetings: list[dict[str, Any]]) -> None: def write_sanctions(run_dir: Path, result: SocietyResult) -> None: - """Tranche 12i: sanctions.jsonl — every sanction, appeal, and rights + """sanctions.jsonl — every sanction, appeal, and rights restoration event in one auditable stream, each line carrying the tick and full detail. Previously these were collected in memory and only surfaced as counts in metrics.json, losing the individual records.""" @@ -2039,7 +2038,7 @@ def write_sanctions(run_dir: Path, result: SocietyResult) -> None: def write_action_reasons(run_dir: Path, result: SocietyResult) -> None: - """Tranche 12i: action_reasons.jsonl — the agent's stated reason for each + """action_reasons.jsonl — the agent's stated reason for each ledgered action, keyed by event_id. Kept as a SIDE CAR (not in the event payload) so the hash-chained event ledger and its frozen byte-identity replay tapes stay untouched while a scientist can still join @@ -2054,7 +2053,7 @@ def write_action_reasons(run_dir: Path, result: SocietyResult) -> None: def write_resource_timeline(run_dir: Path, result: SocietyResult) -> None: - """Tranche 12i: resources.jsonl — per-tick world + per-agent balances, + """resources.jsonl — per-tick world + per-agent balances, a plot-ready time series for studying how resources flow over the campaign (previously only final balances were persisted).""" if not result.resource_timeline: @@ -2066,7 +2065,7 @@ def write_resource_timeline(run_dir: Path, result: SocietyResult) -> None: def write_constitution_history(run_dir: Path, versions: list[str], meetings: list[dict[str, Any]] | None = None, texts: list[dict[str, Any]] | None = None) -> None: - """Tranche 12h: include the amendment payload (what actually changed) + """Include the amendment payload (what actually changed) alongside each version, so the history is human-readable without cross-referencing meetings.jsonl.""" # Map enacted_version -> proposal details from meetings @@ -2081,7 +2080,7 @@ def write_constitution_history(run_dir: Path, versions: list[str], meetings: lis entry: dict[str, Any] = {"version": v} if v in version_payloads: entry["amendments"] = version_payloads[v] - # Tranche 12i: full resulting text (rules + policies) so the + # Full resulting text (rules + policies) so the # law at each version is readable without reconstructing. if texts and i < len(texts): entry["constitution"] = texts[i] @@ -2122,7 +2121,7 @@ def write_timeline(run_dir: Path, result: SocietyResult) -> None: ] for ev in result.events: lines.append(f"- event {ev.event_id} tick={ev.tick} kind={ev.kind}") - # Tranche 12h: assembly details live in meetings.jsonl (one canonical + # Assembly details live in meetings.jsonl (one canonical # file). timeline.md is a thin human-readable index, not a duplicate. if result.meetings: lines += ["", "## Assemblies (see meetings.jsonl for full texts + votes)"] diff --git a/src/verdi/society/inner_life.py b/src/verdi_agents/society/inner_life.py similarity index 100% rename from src/verdi/society/inner_life.py rename to src/verdi_agents/society/inner_life.py diff --git a/src/verdi/society/interview.py b/src/verdi_agents/society/interview.py similarity index 97% rename from src/verdi/society/interview.py rename to src/verdi_agents/society/interview.py index 64e02c6..b8d02d4 100644 --- a/src/verdi/society/interview.py +++ b/src/verdi_agents/society/interview.py @@ -10,7 +10,7 @@ Records are appended to ``run_dir/interviews.jsonl`` (one JSON object per line) and guarded against key material before writing (mirrors the existing artifact -safety in :mod:`verdi.audit.judge_live`). +safety in :mod:`verdi_agents.audit.judge`). """ from __future__ import annotations @@ -20,9 +20,9 @@ from pathlib import Path from typing import Any -from verdi.audit.judge_live import _assert_no_key_material -from verdi.schemas.world import AgentState -from verdi.society.memory import AgentMemory +from verdi_agents.audit.judge import _assert_no_key_material +from verdi_agents.schemas.world import AgentState +from verdi_agents.society.memory import AgentMemory async def _run_bounded(concurrency: int, coros: list) -> list: diff --git a/src/verdi/society/live.py b/src/verdi_agents/society/live.py similarity index 92% rename from src/verdi/society/live.py rename to src/verdi_agents/society/live.py index d10bac6..fc4d5c6 100644 --- a/src/verdi/society/live.py +++ b/src/verdi_agents/society/live.py @@ -1,7 +1,7 @@ """Live entrypoint for a society run (real Qwen, vLLM endpoint). -Wires the real pooled :class:`~verdi.agents.client.ModelClient` into -:class:`~verdi.society.campaign.SocietyCampaign`. Credentials come +Wires the real pooled :class:`~verdi_agents.agents.client.ModelClient` into +:class:`~verdi_agents.society.campaign.SocietyCampaign`. Credentials come exclusively from the process environment (``VERDI_VLLM_*``); a ``.env`` file is loaded via the canonical loader when required (no python-dotenv). Concurrency bootstrap is 4 (``VERDI_MAX_CONCURRENCY``); step-up is manual and @@ -9,7 +9,7 @@ Usage:: - python -m verdi.society.live --scenario examples/starter.yaml \ + python -m verdi_agents.society.live --scenario examples/starter.yaml \ --run-dir runs/my-run [--seed 42] [--concurrency 4] On completion the run auto-generates ``/report.html`` (a local, @@ -31,15 +31,15 @@ from datetime import datetime, timezone from pathlib import Path -from verdi.agents.client import _BACKOFF_CAP, _CONNECT_TIMEOUT, _POOL_TIMEOUT, ModelClient -from verdi.analysis.scenario import load_scenario -from verdi.audit.judge_live import _assert_no_key_material -from verdi.config import ModelEndpointConfig, api_key_values, model_endpoint_from_env -from verdi.experiments.g5_judge_live import _load_dotenv_into_environ -from verdi.society.campaign import PROTOCOL_VERSION, SocietyCampaign +from verdi_agents.agents.client import _BACKOFF_CAP, _CONNECT_TIMEOUT, _POOL_TIMEOUT, ModelClient +from verdi_agents.analysis.scenario import load_scenario +from verdi_agents.audit.judge import _assert_no_key_material +from verdi_agents.config import ModelEndpointConfig, api_key_values, model_endpoint_from_env +from verdi_agents.experiments.pair_judge import _load_dotenv_into_environ +from verdi_agents.society.campaign import PROTOCOL_VERSION, SocietyCampaign DEFAULT_CONCURRENCY = 4 -# Per-model-call liveness watchdog. Calibrated per Tranche 12d review F1: +# Per-model-call liveness watchdog. Calibration: # per-call beats (model_calls.jsonl, written after every COMPLETED model # call) reset the deadline continuously, so the window only needs to outlast # a SINGLE stuck call: VERDI_REQUEST_TIMEOUT (180s) x (max_retries+1) (6) = @@ -47,10 +47,10 @@ # that with margin; shorter windows could trip while the client is still # legitimately retrying a dead call. The old events-only 600s design # false-fired on whole assembly ticks (zero ledger writes until finalize) -# and killed the live C-1 run at t15 — see state/LESSONS.md Tranche 12d. +# and killed a 24-agent live run at its first assembly (t15). DEFAULT_STALL_TIMEOUT_SECONDS = 1200 _HEARTBEAT_INTERVAL_SECONDS = 20 -#: Tranche 12i: model_calls.jsonl is the single per-call log (timestamps + +#: model_calls.jsonl is the single per-call log (timestamps + #: tokens + duration + context). The old progress.jsonl beat file is gone; #: liveness is measured from model_calls.jsonl line growth. _MODEL_CALLS_FILE = "model_calls.jsonl" @@ -68,8 +68,8 @@ def _progress_count(run_dir: Path) -> int: A full assembly tick appends nothing to events.jsonl until its very end (deliberation + voting + finalize all precede the ledger commit), so - event count alone false-fires on legitimately long ticks (Tranche 12d - review F1: killed the live C-1 run at its first assembly, t15). + event count alone false-fires on legitimately long ticks (it once + killed a 24-agent live run at its first assembly, t15). model_calls.jsonl gains a line after every COMPLETED model call (all purposes: action, reflection, assembly, probe), so the deadline now resets continuously during long ticks while still tripping on a stuck @@ -86,7 +86,7 @@ async def _stall_watchdog(run_dir: Path, stall_timeout_seconds: float, A hung endpoint can hold a socket open without ever responding, which parks the whole campaign on one in-flight call while httpx's timeout - (which only fires on read activity) never triggers (Tranche 12a lesson). + (which only fires on read activity) never triggers. This watchdog watches wall-clock progress of the COMBINED event + per-model-call beat count: it writes a heartbeat to run_meta on a short interval, and if that count does not grow for ``stall_timeout_seconds`` @@ -205,7 +205,7 @@ async def _run(scenario_path: Path, run_dir: Path, seed: int | None, concurrency return 2 print(f"smoke OK: model={smoke['model']} trace={smoke['trace_id']}") - # Tranche 12d F1 / 12i: model_calls.jsonl (written per completed + # model_calls.jsonl (written per completed # call by the campaign) is the liveness signal, so long assembly # ticks (zero event writes until finalize) don't false-fire the # watchdog. No wrapper needed — the campaign writes the file. @@ -292,7 +292,7 @@ def _export_html_report(run_dir: Path) -> str: tile URLs). """ try: - from verdi.analysis.html_report import write_report + from verdi_agents.analysis.html_report import write_report out = write_report(run_dir, run_dir / "report.html") _guard_report_clean(out.read_text(encoding="utf-8")) return f"report: {out} ({out.stat().st_size // 1024} KB) — open in a browser" @@ -324,7 +324,7 @@ def _guard_artifacts_clean(run_dir: Path) -> None: def _write_run_meta(run_dir: Path, obj: dict) -> None: """Merge ``obj`` into run_meta.json atomically. - Tranche 12d review F2: the watchdog and the main flow are two writers; a + The watchdog and the main flow are two writers; a temp-file + os.replace keeps the file whole under any interleave (a crash mid-write can no longer truncate it) even though, on a single event loop, the synchronous read-modify-write is already GIL-atomic per call. @@ -344,7 +344,7 @@ def _write_run_meta(run_dir: Path, obj: dict) -> None: def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Run a Society V2 live campaign.") + parser = argparse.ArgumentParser(description="Run a live society campaign.") parser.add_argument("--scenario", required=True, type=Path, help="scenario JSON path") parser.add_argument("--run-dir", required=True, type=Path, help="artifact output directory") parser.add_argument("--seed", type=int, default=None, help="run seed (default: scenario run_seed)") diff --git a/src/verdi/society/memory.py b/src/verdi_agents/society/memory.py similarity index 100% rename from src/verdi/society/memory.py rename to src/verdi_agents/society/memory.py diff --git a/src/verdi/society/prompting.py b/src/verdi_agents/society/prompting.py similarity index 97% rename from src/verdi/society/prompting.py rename to src/verdi_agents/society/prompting.py index 4e59155..905b5c0 100644 --- a/src/verdi/society/prompting.py +++ b/src/verdi_agents/society/prompting.py @@ -2,8 +2,7 @@ Every social prompt embeds the agent's ``AgentMemory.render_history_summary`` output, which is derived from committed events only. Static prompts that -ignore committed state are a protocol violation -(docs/SOCIETY_V2_PROTOCOL.md, section 0 gap 2 and section 3). +ignore committed state are a protocol violation. The builder is pure: no clock, no network, no hidden-probe content (probes live in a separate package and are never referenced here). @@ -12,7 +11,7 @@ import json -from verdi.schemas import Task, WorldState +from verdi_agents.schemas import Task, WorldState from .memory import AgentMemory @@ -20,7 +19,7 @@ SOCIAL_PROMPT_VERSION = "verdi.society-prompt/5" #: Exact required argument field names per action kind, mirroring -#: ``verdi.world.validate``. Rendered into the prompt so the model cannot +#: ``verdi_agents.world.validate``. Rendered into the prompt so the model cannot #: confuse ``amount`` (deposit/withdraw) with ``units`` (transfer) or omit #: ``text`` on a signal. This schema, not free constraint keys, is the #: contract the model is told to follow. diff --git a/src/verdi/society/tasks.py b/src/verdi_agents/society/tasks.py similarity index 99% rename from src/verdi/society/tasks.py rename to src/verdi_agents/society/tasks.py index 57b9f93..488efab 100644 --- a/src/verdi/society/tasks.py +++ b/src/verdi_agents/society/tasks.py @@ -22,15 +22,15 @@ import random from typing import TYPE_CHECKING -from verdi.schemas import Task +from verdi_agents.schemas import Task from .memory import ADVERSARY_MARKER if TYPE_CHECKING: - from verdi.schemas import AgentState, WorldState + from verdi_agents.schemas import AgentState, WorldState #: Accepted action kinds that mutate substantive state -#: (mirrors WorldReducer.MUTATING_KINDS, src/verdi/world/reducer.py:39). +#: (mirrors WorldReducer.MUTATING_KINDS, src/verdi_agents/world/reducer.py:39). MUTATING_KINDS = frozenset({"transfer", "deposit", "withdraw", "produce", "emit_signal"}) #: Default amount for honest-agent resource tasks. @@ -180,7 +180,7 @@ def generate_tick_tape( # argument field names (validate.py: transfer takes # to/resource/units). An earlier version used "amount" here, # which leaked into live runs as 100% missing_argument - # rejections (Tranche 11 lesson). + # rejections. constraints = {"resource": resource, "units": amount, "to": target} counterpart_ids = [target] public_observations = [f"transfer {amount} {resource} to {target}"] diff --git a/src/verdi_agents/sweep.py b/src/verdi_agents/sweep.py new file mode 100644 index 0000000..71f3b2a --- /dev/null +++ b/src/verdi_agents/sweep.py @@ -0,0 +1,5 @@ +"""``python -m verdi_agents.sweep``: see :mod:`verdi_agents.experiments.sweep`.""" +from verdi_agents.experiments.sweep import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/verdi/world/__init__.py b/src/verdi_agents/world/__init__.py similarity index 71% rename from src/verdi/world/__init__.py rename to src/verdi_agents/world/__init__.py index 9e34e0e..146ddfe 100644 --- a/src/verdi/world/__init__.py +++ b/src/verdi_agents/world/__init__.py @@ -1,4 +1,4 @@ -"""Deterministic, event-sourced world engine (Worker A, P1-A). +"""Deterministic, event-sourced world engine. Public API ---------- @@ -13,16 +13,16 @@ """ from __future__ import annotations -from verdi.world.canonical import canonical_json, canonical_sha256 -from verdi.world.log import EventLog, LogIntegrityError -from verdi.world.reducer import ( +from verdi_agents.world.canonical import canonical_json, canonical_sha256 +from verdi_agents.world.log import EventLog, LogIntegrityError +from verdi_agents.world.reducer import ( GENESIS_HASH, InvariantViolationError, WorldReducer, canonical_state_hash, ) -from verdi.world.replay import TapeStep, replay, replay_in_memory -from verdi.world.snapshot import SnapshotError, SnapshotStore +from verdi_agents.world.replay import TapeStep, replay, replay_in_memory +from verdi_agents.world.snapshot import SnapshotError, SnapshotStore __all__ = [ "EventLog", diff --git a/src/verdi/world/branch.py b/src/verdi_agents/world/branch.py similarity index 87% rename from src/verdi/world/branch.py rename to src/verdi_agents/world/branch.py index fb147f7..506083a 100644 --- a/src/verdi/world/branch.py +++ b/src/verdi_agents/world/branch.py @@ -1,12 +1,11 @@ -"""Counterfactual snapshot branching (P2 WS-1, frozen contract v1.0). +"""Fork a new ledger from a saved world snapshot to replay a counterfactual branch. A *branch* forks a new ledger from a sealed point of the world: a -:class:`~verdi.schemas.WorldState` snapshot plus the hash-chained prefix +:class:`~verdi_agents.schemas.WorldState` snapshot plus the hash-chained prefix ledger up to the branch point. The counterfactual tape (the *remaining* -recorded :class:`~verdi.world.replay.TapeStep` sequence) is folded through -the same deterministic reducer the pilot uses, so the branch is a -replay-only construction: no model invocation, fully deterministic -(ADR-004, ADR-008). +recorded :class:`~verdi_agents.world.replay.TapeStep` sequence) is folded through +the same deterministic reducer the reference run uses, so the branch is a +replay-only construction: no model invocation, fully deterministic. Guarantees (tested in ``tests/world/test_branch.py``): * B1: inputs (snapshot, prefix log) are never mutated; @@ -29,10 +28,10 @@ from pathlib import Path from typing import Any -from verdi.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState -from verdi.world.log import EventLog -from verdi.world.reducer import GENESIS_HASH, WorldReducer -from verdi.world.replay import TapeStep +from verdi_agents.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import GENESIS_HASH, WorldReducer +from verdi_agents.world.replay import TapeStep class BranchOriginError(RuntimeError): @@ -45,9 +44,9 @@ class BranchSpec: origin_state_hash: str # canonical_state_hash of the snapshot origin_event_count: int # ledger length at the branch point - # Caller-supplied LIFECYCLE-tick metadata at the branch point. Per - # ADR-008 the reducer tick is a pure transition counter, not the - # pilot's lifecycle tick, so this is identity/provenance metadata + # Caller-supplied LIFECYCLE-tick metadata at the branch point. The + # reducer tick is a pure transition counter, not the + # reference run's lifecycle tick, so this is identity/provenance metadata # only — branch_from_snapshot performs no cross-check on it. origin_tick: int branch_id: str # e.g. "counterfactual-v2" @@ -65,7 +64,7 @@ class BranchResult: # --------------------------------------------------------------------------- # -# governance event construction (ported from the pilot's replay path) +# governance event construction (ported from the reference run's replay path) # --------------------------------------------------------------------------- # def _governance_event( before: WorldState, @@ -77,10 +76,10 @@ def _governance_event( """Unsealed governance ledger event, byte-identical to the action-path construction except for ``kind``/``payload``. - Local copy of the pilot's ``build_governance_event`` so the world - engine has no dependency on ``verdi.pilot``. Keep in sync with its - counterpart in ``verdi.pilot.driver.build_governance_event``; the - pilot-branch byte-identity test + Local copy of the reference run's ``build_governance_event`` so the world + engine has no dependency on ``verdi_agents.reference_run``. Keep in sync with its + counterpart in ``verdi_agents.reference_run.driver.build_governance_event``; the + reference-run/branch byte-identity test (``tests/world/test_branch.py::test_branch_replay_matches_live_ledger``) guards against drift between the two builders. ``event_id`` comes from ``before.event_cursor`` (same as the reducer); ``prev_hash`` is @@ -133,7 +132,7 @@ def branch_from_snapshot( ) -> BranchResult: """Fold ``remaining_tape`` from a sealed snapshot into a branch ledger. - The fold uses the exact pilot replay semantics (governance steps take + The fold uses the exact reference-run replay semantics (governance steps take their ``kind``/``payload`` from ``step.governance``) and CONTINUES the hash chain: the first folded event has ``prev_hash = log.last_hash()`` and ``event_id = len(log) + i``. @@ -150,7 +149,7 @@ def branch_from_snapshot( spec: Branch origin + treatment identity (see :class:`BranchSpec`). ``spec.origin_tick`` is caller-supplied LIFECYCLE-tick metadata - only (per ADR-008 the reducer tick is a transition counter, so + only (the reducer tick is a transition counter, so no hard cross-check against the reducer tick is possible); it is carried through to the result but never validated here. @@ -234,7 +233,7 @@ def branch_from_snapshot( def manifest_for_branch(spec: BranchSpec, base_manifest: dict[str, Any]) -> dict[str, Any]: """Return ``base_manifest`` with branch identity applied (B5). - Schema-free helper: the pilot owns :class:`ExperimentManifest` + Schema-free helper: the reference run owns :class:`ExperimentManifest` construction; this only guarantees ``branch_origin`` / ``treatment_seed`` carry through. The input dict is not mutated. """ diff --git a/src/verdi/world/canonical.py b/src/verdi_agents/world/canonical.py similarity index 100% rename from src/verdi/world/canonical.py rename to src/verdi_agents/world/canonical.py diff --git a/src/verdi/world/economy.py b/src/verdi_agents/world/economy.py similarity index 99% rename from src/verdi/world/economy.py rename to src/verdi_agents/world/economy.py index b21dcd9..04ff544 100644 --- a/src/verdi/world/economy.py +++ b/src/verdi_agents/world/economy.py @@ -1,7 +1,7 @@ """Opt-in versioned constrained economy + objective task-success oracle. -This module is the DETERMINISTIC core of the P0-B "meaningful shock and -task environment" work: pure, stateless functions that enforce a finite +This module is the DETERMINISTIC core of the economy and task +environment: pure, stateless functions that enforce a finite economy (per-tick production caps, production input costs, end-of-tick consumption) and evaluate verifiable delivery objectives from committed world state alone. Nothing here calls a model, reads the wall clock, or @@ -49,7 +49,7 @@ from dataclasses import dataclass from typing import Any -from verdi.schemas import WorldState +from verdi_agents.schemas import WorldState #: Economy spec versions this engine understands. ECONOMY_SPEC_VERSIONS = frozenset({1}) diff --git a/src/verdi/world/log.py b/src/verdi_agents/world/log.py similarity index 97% rename from src/verdi/world/log.py rename to src/verdi_agents/world/log.py index 431987c..e7fdbf6 100644 --- a/src/verdi/world/log.py +++ b/src/verdi_agents/world/log.py @@ -18,9 +18,9 @@ from collections.abc import Iterator from pathlib import Path -from verdi.schemas import WorldEvent -from verdi.world.canonical import canonical_json, canonical_sha256 -from verdi.world.reducer import GENESIS_HASH +from verdi_agents.schemas import WorldEvent +from verdi_agents.world.canonical import canonical_json, canonical_sha256 +from verdi_agents.world.reducer import GENESIS_HASH class LogIntegrityError(RuntimeError): diff --git a/src/verdi/world/observation.py b/src/verdi_agents/world/observation.py similarity index 94% rename from src/verdi/world/observation.py rename to src/verdi_agents/world/observation.py index ab9f5cf..99b01a1 100644 --- a/src/verdi/world/observation.py +++ b/src/verdi_agents/world/observation.py @@ -1,9 +1,8 @@ -"""Pure per-agent observation of committed world state (Lane W). +"""Pure per-agent observation of committed world state. ``agent_observation`` is a pure function of a committed :class:`WorldState`: it never reads the future, never writes, never uses wall clock or global -RNG. This purity is what makes the feedback-leakage tests possible -(SOCIETY_V2_PROTOCOL.md §2, Lane W invariants). +RNG. This purity is what makes the feedback-leakage tests possible. """ from __future__ import annotations @@ -11,7 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field -from verdi.schemas import RightsState, WorldState +from verdi_agents.schemas import RightsState, WorldState #: All action kinds that mutate substantive state (mirror of #: ``WorldReducer.MUTATING_KINDS``). diff --git a/src/verdi/world/policy.py b/src/verdi_agents/world/policy.py similarity index 100% rename from src/verdi/world/policy.py rename to src/verdi_agents/world/policy.py diff --git a/src/verdi/world/reducer.py b/src/verdi_agents/world/reducer.py similarity index 97% rename from src/verdi/world/reducer.py rename to src/verdi_agents/world/reducer.py index 1c67a02..b0f8e26 100644 --- a/src/verdi/world/reducer.py +++ b/src/verdi_agents/world/reducer.py @@ -29,8 +29,8 @@ from typing import Any -from verdi.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState -from verdi.world.canonical import canonical_sha256 +from verdi_agents.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState +from verdi_agents.world.canonical import canonical_sha256 GENESIS_HASH = "genesis" @@ -82,7 +82,7 @@ def apply( return self._with_hash(nxt) if action.action_kind == "task_objective" or action.actor_id == "world": - # Deterministic record-only world steps (P0-B delivery-objective + # Deterministic record-only world steps (delivery-objective # outcomes): content is unchanged by construction (before == # after); only the ledger bookkeeping advances. The campaign's # replay path rebuilds these events from the recorded tape @@ -129,7 +129,7 @@ def build_event( ) -> WorldEvent: """Build the ledger event for one transition (accepted or rejected). - ``event_hash`` is left empty: :class:`~verdi.world.log.EventLog` + ``event_hash`` is left empty: :class:`~verdi_agents.world.log.EventLog` is the single writer that fills it on append so the chain is computed in exactly one place. """ diff --git a/src/verdi/world/replay.py b/src/verdi_agents/world/replay.py similarity index 91% rename from src/verdi/world/replay.py rename to src/verdi_agents/world/replay.py index 75584a6..8b7d127 100644 --- a/src/verdi/world/replay.py +++ b/src/verdi_agents/world/replay.py @@ -4,15 +4,15 @@ the *tape* -- through the deterministic reducer and the hash-chained event log. It performs NO model invocation: everything is read from the recorded outputs, so two replays over the same tape yield an identical -final state hash and an identical event list (G3). +final state hash and an identical event list. """ from __future__ import annotations from dataclasses import dataclass -from verdi.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState -from verdi.world.log import EventLog -from verdi.world.reducer import WorldReducer +from verdi_agents.schemas import ProposedAction, ValidationResult, WorldEvent, WorldState +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import WorldReducer @dataclass(frozen=True) @@ -41,11 +41,11 @@ class TapeStep: # position; takes no action, advances no ledger event or tick — so a live # run and its replay both apply it identically and the state hash matches. shock: dict | None = None - # P0-B (constrained economy): deterministic production cost deduction + # Constrained economy: deterministic production cost deduction # ({actor_id, resource, amount}) applied at this position before the # produce commit; state-only, like rights_change / world_shock. production: dict | None = None - # P0-B: end-of-tick consumption payload ({tick_consumed, deficit_agents}) + # End-of-tick consumption payload ({tick_consumed, deficit_agents}) # recorded so replay applies the same deterministic deduction and rebuilds # the identical ledger event. consumption: dict | None = None diff --git a/src/verdi/world/snapshot.py b/src/verdi_agents/world/snapshot.py similarity index 96% rename from src/verdi/world/snapshot.py rename to src/verdi_agents/world/snapshot.py index 411eecc..44d0828 100644 --- a/src/verdi/world/snapshot.py +++ b/src/verdi_agents/world/snapshot.py @@ -11,8 +11,8 @@ from pydantic import ValidationError -from verdi.schemas import WorldState -from verdi.world.canonical import canonical_json, canonical_sha256 +from verdi_agents.schemas import WorldState +from verdi_agents.world.canonical import canonical_json, canonical_sha256 class SnapshotError(RuntimeError): diff --git a/src/verdi/world/validate.py b/src/verdi_agents/world/validate.py similarity index 97% rename from src/verdi/world/validate.py rename to src/verdi_agents/world/validate.py index 98d12ea..69029f9 100644 --- a/src/verdi/world/validate.py +++ b/src/verdi_agents/world/validate.py @@ -1,6 +1,6 @@ -"""Deterministic action validator (Lane W). +"""Deterministic action validator. -Replaces the pilot's hardcoded ``accepted=True``. Pure and deterministic: +Replaces the reference run's hardcoded ``accepted=True``. Pure and deterministic: no model call, no wall clock, no RNG. Inspects the committed :class:`WorldState` and the :class:`ProposedAction` and returns a :class:`ValidationResult`. Never mutates state; a rejection must be @@ -12,9 +12,9 @@ from collections.abc import Mapping from typing import Any -from verdi.schemas import ProposedAction, ValidationResult, WorldState -from verdi.world.economy import EconomySpec, validate_production -from verdi.world.policy import policy_block +from verdi_agents.schemas import ProposedAction, ValidationResult, WorldState +from verdi_agents.world.economy import EconomySpec, validate_production +from verdi_agents.world.policy import policy_block #: Kinds whose property-moving capability is withdrawn under sanction. SANCTION_BLOCKED_KINDS = frozenset({"transfer", "deposit"}) diff --git a/tests/adapters/test_concordia_reuse.py b/tests/adapters/test_concordia_reuse.py index fd033c3..11448eb 100644 --- a/tests/adapters/test_concordia_reuse.py +++ b/tests/adapters/test_concordia_reuse.py @@ -1,4 +1,4 @@ -"""Tests for the Concordia deep-reuse adapter (todo D). +"""Tests for the Concordia deep-reuse adapter. These are skip-guarded: they run only when ``gdm-concordia`` is installed. The module under test is lazy and dependency-free at import, so importing it always @@ -11,7 +11,7 @@ pytest.importorskip("concordia") -from verdi.adapters import concordia_reuse as cr # noqa: E402 +from verdi_agents.adapters import concordia_reuse as cr # noqa: E402 def test_value_axes_constant(): diff --git a/tests/agents/fakes.py b/tests/agents/fakes.py index 7cd2ae1..e512455 100644 --- a/tests/agents/fakes.py +++ b/tests/agents/fakes.py @@ -5,7 +5,7 @@ import httpx -from verdi.config import ModelEndpointConfig +from verdi_agents.config import ModelEndpointConfig def make_config(**overrides) -> ModelEndpointConfig: diff --git a/tests/agents/test_audit_regressions.py b/tests/agents/test_audit_regressions.py index d72ebb8..a695b16 100644 --- a/tests/agents/test_audit_regressions.py +++ b/tests/agents/test_audit_regressions.py @@ -1,4 +1,4 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import asyncio @@ -6,9 +6,9 @@ import httpx import pytest -from verdi import config as cfg_mod -from verdi.agents import ModelClient -from verdi.agents.client import ModelError +from verdi_agents import config as cfg_mod +from verdi_agents.agents import ModelClient +from verdi_agents.agents.client import ModelError from .fakes import completion_body, make_config diff --git a/tests/agents/test_client.py b/tests/agents/test_client.py index 00e186d..4d74e2d 100644 --- a/tests/agents/test_client.py +++ b/tests/agents/test_client.py @@ -3,8 +3,8 @@ import pytest -from verdi.agents import ModelClient -from verdi.agents.client import ModelError +from verdi_agents.agents import ModelClient +from verdi_agents.agents.client import ModelError from .fakes import chat_transport, completion_body, make_config @@ -59,7 +59,7 @@ async def run(): def test_524_origin_timeout_is_retried(): - """LESSONS 12g: C-1 attempt 6 died on an un-retried HTTP 524 + """Regression: an earlier live run died on an un-retried HTTP 524 (Cloudflare origin-timeout from a slow 27B origin). 524 must retry like 502/503 and recover on the next attempt.""" records: list[dict] = [] diff --git a/tests/agents/test_client_soak.py b/tests/agents/test_client_soak.py index d26b0a8..9aa85fd 100644 --- a/tests/agents/test_client_soak.py +++ b/tests/agents/test_client_soak.py @@ -14,8 +14,8 @@ import httpx import pytest -from verdi.agents import ModelClient -from verdi.agents.client import _BACKOFF_BASE, _BACKOFF_CAP, ModelError, ModelResult +from verdi_agents.agents import ModelClient +from verdi_agents.agents.client import _BACKOFF_BASE, _BACKOFF_CAP, ModelError, ModelResult from .fakes import chat_transport, completion_body, make_config @@ -49,7 +49,7 @@ def fake_uuid4() -> uuid_module.UUID: counter += 1 return uuid_module.UUID(value) - monkeypatch.setattr("verdi.agents.client.uuid.uuid4", fake_uuid4) + monkeypatch.setattr("verdi_agents.agents.client.uuid.uuid4", fake_uuid4) return generated diff --git a/tests/agents/test_policy.py b/tests/agents/test_policy.py index 99a66ed..1efa121 100644 --- a/tests/agents/test_policy.py +++ b/tests/agents/test_policy.py @@ -3,9 +3,9 @@ import pytest -from verdi.agents import PROMPT_VERSION, AgentPolicy -from verdi.agents.client import ModelResult -from verdi.schemas import Task +from verdi_agents.agents import PROMPT_VERSION, AgentPolicy +from verdi_agents.agents.client import ModelResult +from verdi_agents.schemas import Task class CannedModel: diff --git a/tests/agents/test_pool.py b/tests/agents/test_pool.py index 860f523..8f01d9e 100644 --- a/tests/agents/test_pool.py +++ b/tests/agents/test_pool.py @@ -3,7 +3,7 @@ import pytest -from verdi.agents import AgentPolicy, TaskPool, make_task +from verdi_agents.agents import AgentPolicy, TaskPool, make_task class CannedModel: @@ -14,7 +14,7 @@ async def generate_structured(self, messages, *, response_format_json=False, tem import asyncio as _a await _a.sleep(0.001) - from verdi.agents.client import ModelResult + from verdi_agents.agents.client import ModelResult return ModelResult(content=self.content, request_id="r", model="c", trace_id="t") @@ -61,7 +61,7 @@ async def worker(task): await _a.sleep(0.002) current -= 1 - from verdi.agents import PolicyResult + from verdi_agents.agents import PolicyResult return PolicyResult(action=None, quarantine_reason="counting only") diff --git a/tests/agents/test_recorded.py b/tests/agents/test_recorded.py index cf9bbdf..0cac8da 100644 --- a/tests/agents/test_recorded.py +++ b/tests/agents/test_recorded.py @@ -3,11 +3,11 @@ import pytest -from verdi.agents import AgentPolicy, RecordedModel, TapeExhausted +from verdi_agents.agents import AgentPolicy, RecordedModel, TapeExhausted def make_task(task_id: str, actor_id: str = "a1"): - from verdi.agents import make_task as _mt + from verdi_agents.agents import make_task as _mt return _mt(task_id, actor_id) diff --git a/tests/agents/test_reliability.py b/tests/agents/test_reliability.py index b1f263d..2d1a311 100644 --- a/tests/agents/test_reliability.py +++ b/tests/agents/test_reliability.py @@ -1,8 +1,8 @@ -"""P3 reliability soak for the ModelClient retry path (WS-6). +"""Reliability soak tests for the ModelClient retry path. Deterministic, offline, no wall-clock sleep: ``asyncio.sleep`` is replaced by a recorder and the backoff RNG is seeded per scenario. Expected values are -recomputed from ``verdi.agents.client`` constants (``_BACKOFF_BASE``, +recomputed from ``verdi_agents.agents.client`` constants (``_BACKOFF_BASE``, ``_BACKOFF_CAP``, ``_backoff_delay``), never hardcoded. """ from __future__ import annotations @@ -14,8 +14,8 @@ import httpx import pytest -from verdi.agents import ModelClient -from verdi.agents.client import _BACKOFF_BASE, _BACKOFF_CAP, ModelError, _backoff_delay +from verdi_agents.agents import ModelClient +from verdi_agents.agents.client import _BACKOFF_BASE, _BACKOFF_CAP, ModelError, _backoff_delay from .fakes import completion_body, make_config diff --git a/tests/analysis/conftest.py b/tests/analysis/conftest.py index f9b7def..7105067 100644 --- a/tests/analysis/conftest.py +++ b/tests/analysis/conftest.py @@ -1,4 +1,4 @@ -"""Shared builders for analysis tests (P1-E).""" +"""Shared builders for analysis tests.""" from __future__ import annotations import json @@ -6,7 +6,7 @@ import pytest -from verdi.schemas import ( +from verdi_agents.schemas import ( Appeal, EvidenceReference, ProbeResult, @@ -16,8 +16,8 @@ ) REPO_ROOT = Path(__file__).resolve().parent.parent.parent -SCENARIO = REPO_ROOT / "experiments" / "scenarios" / "pilot_v1.json" -TAPE = REPO_ROOT / "experiments" / "task_tape" / "pilot_v1.json" +SCENARIO = REPO_ROOT / "tests" / "fixtures" / "reference_run" / "scenarios" / "pilot_v1.json" +TAPE = REPO_ROOT / "tests" / "fixtures" / "reference_run" / "task_tape" / "pilot_v1.json" def probe(agent: str, tick: int, channel: str, vector: dict[str, float], pid: str | None = None) -> ProbeResult: @@ -60,7 +60,7 @@ def event(eid: int, tick: int, kind: str, decision: str, payload: dict, prev: st def world_state(tick: int, resources_by_agent: dict[str, dict[str, float]]) -> WorldState: - from verdi.schemas import AgentState, ValueProfile + from verdi_agents.schemas import AgentState, ValueProfile values = ValueProfile(integrity=0.5, fairness=0.5, cooperation=0.5, autonomy=0.5, solidarity=0.5) agents = { diff --git a/tests/analysis/test_audit_regressions.py b/tests/analysis/test_audit_regressions.py index c3209fa..39c61c5 100644 --- a/tests/analysis/test_audit_regressions.py +++ b/tests/analysis/test_audit_regressions.py @@ -1,4 +1,4 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import json @@ -6,7 +6,7 @@ import pytest import yaml -from verdi.analysis.outcome_metrics import recovery_time +from verdi_agents.analysis.outcome_metrics import recovery_time def test_recovery_time_maps_outcomes_to_due_ticks_in_evaluation_order(tmp_path): @@ -50,7 +50,7 @@ def _imposed(sid, subject, tick): def test_open_exclusion_ends_at_last_lifecycle_tick_not_ledger_tick(tmp_path): - from verdi.analysis.outcome_metrics import exclusion_episodes + from verdi_agents.analysis.outcome_metrics import exclusion_episodes _write_run(tmp_path, [_imposed("s1", "grace", 19)], range(20), [1, 50, 104]) # bounded by the last LIFECYCLE tick (19), not the ledger counter (104 -> 85) @@ -58,7 +58,7 @@ def test_open_exclusion_ends_at_last_lifecycle_tick_not_ledger_tick(tmp_path): def test_overlapping_exclusions_are_merged(tmp_path): - from verdi.analysis.outcome_metrics import exclusion_episodes + from verdi_agents.analysis.outcome_metrics import exclusion_episodes _write_run(tmp_path, [ _imposed("s1", "dave", 0), _imposed("s2", "dave", 2), @@ -71,7 +71,7 @@ def test_overlapping_exclusions_are_merged(tmp_path): def test_comparative_report_marks_undefined_baseline_not_evaluable(tmp_path): - from verdi.analysis.comparative_report import _recovery_row + from verdi_agents.analysis.comparative_report import _recovery_row (tmp_path / "events.jsonl").write_text( json.dumps({"event_id": 1, "tick": 31, "kind": "task_completed"}) + "\n") @@ -82,7 +82,7 @@ def test_comparative_report_marks_undefined_baseline_not_evaluable(tmp_path): def test_completion_due_tick_is_lifecycle_tick_not_ledger_tick(tmp_path): - from verdi.analysis.outcome_metrics import completion_rate + from verdi_agents.analysis.outcome_metrics import completion_rate rows = [ {"event_id": 0, "tick": 55, "kind": "action_accepted", @@ -104,7 +104,7 @@ def _scenario(**extra): import copy from pathlib import Path - from verdi.analysis.scenario import load_scenario + from verdi_agents.analysis.scenario import load_scenario sc = copy.deepcopy(load_scenario(Path(__file__).resolve().parents[2] / "examples" / "starter.yaml")) sc.update(extra) @@ -122,7 +122,7 @@ def _scenario(**extra): (lambda sc: sc.update(agents=[]), "no agents"), ]) def test_bad_scenario_values_fail_at_load_time(mutate, message): - from verdi.analysis.scenario import ScenarioLoadError, build_t0_state + from verdi_agents.analysis.scenario import ScenarioLoadError, build_t0_state sc = _scenario() mutate(sc) @@ -131,7 +131,7 @@ def test_bad_scenario_values_fail_at_load_time(mutate, message): def test_completion_due_tick_counts_consumption_ticks_when_economy_active(tmp_path): - from verdi.analysis.outcome_metrics import completion_rate + from verdi_agents.analysis.outcome_metrics import completion_rate rows = [{"event_id": i, "tick": i + 1, "kind": "consumption"} for i in range(11)] # every tick-10 action was quarantined: no soc-t10 task id in the ledger diff --git a/tests/analysis/test_estimates.py b/tests/analysis/test_estimates.py index 2989f3c..342100b 100644 --- a/tests/analysis/test_estimates.py +++ b/tests/analysis/test_estimates.py @@ -1,9 +1,9 @@ -"""Estimand wiring tests (P2 WS-4, tests/analysis). +"""Tests that each preregistered estimand is computed from the right metric. Covers: -* ``run_estimand_values`` maps the six PREREGISTRATION estimands to the +* ``run_estimand_values`` maps the six preregistered estimands to the metrics.py functions (checked against hand-computed values on a small - synthetic run, and against the pilot's metric outputs); + synthetic run, and against the reference run's metric outputs); * ``paired_estimates`` single-pair honesty (sufficient=False, CI withheld) and multi-pair behavior (real CIs from n_pairs >= 10, insufficient flag for 2-9 pairs). @@ -18,12 +18,12 @@ world_state, ) -from verdi.analysis.estimates import ( +from verdi_agents.analysis.estimates import ( ESTIMANDS, paired_estimates, run_estimand_values, ) -from verdi.analysis.statistics import PairedResult +from verdi_agents.analysis.statistics import PairedResult # --------------------------------------------------------------------------- # # synthetic ledger: 4 action events (1 rejected), 1 sanction, 1 upheld appeal @@ -72,7 +72,7 @@ def test_run_estimand_values_maps_all_six(): assert values["false_punishment_rate"] == pytest.approx(1.0) # terminal snapshot tick 6: mean(3, 7), gini of [3, 7] assert values["welfare_mean_food"] == pytest.approx(5.0) - from verdi.analysis.metrics import gini + from verdi_agents.analysis.metrics import gini assert values["inequality_gini_food"] == pytest.approx(gini([3.0, 7.0])) # restriction end = tick 4 (rights_restored); post ticks 5, 6 both meet @@ -151,13 +151,13 @@ def test_run_dict_missing_estimates_raises(): def test_paired_estimates_empty_raises(): - # SF2b: an empty pair sequence is a caller bug, not an estimate. + # An empty pair sequence is a caller bug, not an estimate. with pytest.raises(ValueError, match="at least one matched pair"): paired_estimates([]) # --------------------------------------------------------------------------- # -# SF1: sustained_recovery operational definition (PREREGISTRATION v1.1) +# sustained_recovery operational definition (preregistered, v1.1) # --------------------------------------------------------------------------- # def _single_sanction_events(sid="s1", subject="a-01", end=4): @@ -172,7 +172,7 @@ def _single_sanction_events(sid="s1", subject="a-01", end=4): def test_sustained_recovery_single_crossing_disagrees_with_k_boolean(): - # SF1 fail-before/pass-after: the agent meets the pre-sanction level + # Fails under the old formula, passes under the new one: the agent meets the pre-sanction level # at only ONE of three post-restriction snapshot ticks (ticks 6, 7, 9; # food 1, 1, 3; pre-sanction level 3). # OLD formula: k=1 consecutive-crossing boolean -> no 2 consecutive @@ -197,7 +197,7 @@ def test_sustained_recovery_single_crossing_disagrees_with_k_boolean(): def test_sustained_recovery_mean_of_per_agent_fractions(): - # SF1: two treated agents, each with post-restriction snapshot ticks + # Two treated agents, each with post-restriction snapshot ticks # {5,6,7,8}. a-01 meets its pre-sanction level at tick 5 only (1/4); # a-02 meets at ticks 6 and 8 (2/4). # NEW formula (v1.1 operational): estimand = mean(1/4, 2/4) = 3/8. @@ -232,12 +232,12 @@ def ws(tick, f1, f2): # --------------------------------------------------------------------------- # -# SF2: false_punishment_rate operational definition (PREREGISTRATION v1.1) +# false_punishment_rate operational definition (preregistered, v1.1) # --------------------------------------------------------------------------- # def _sanction_with_restoration_but_dismissed_appeal(): # A sanction whose appeal was DISMISSED but whose rights were later - # restored (rights_restored event): under the PREREG v1.0 ideal + # restored (rights_restored event): under the preregistered v1.0 ideal # definition ("upheld-then-reversed OR appealed-and-restored") this # sanction counts in the numerator via the restoration leg; under # the v1.1 OPERATIONAL definition (appealed-and-upheld / all) it @@ -258,7 +258,7 @@ def _sanction_with_restoration_but_dismissed_appeal(): def test_false_punishment_rate_restoration_leg_disagrees(): - # SF2: restored-but-appeal-dismissed sanction. In the v1.0 ledger + # Restored-but-appeal-dismissed sanction. In the v1.0 ledger # schema there is NO sanction state distinct from "upheld appeal": # the only reversal evidence is a `rights_restored` event, which # this synthetic run carries alongside a DISMISSED appeal. diff --git a/tests/analysis/test_html_report.py b/tests/analysis/test_html_report.py index a107b87..9385e43 100644 --- a/tests/analysis/test_html_report.py +++ b/tests/analysis/test_html_report.py @@ -2,7 +2,7 @@ import pytest -from verdi.analysis.html_report import load_run, safe_json, write_report +from verdi_agents.analysis.html_report import load_run, safe_json, write_report def test_partial_tail_and_missing_streams(tmp_path): @@ -64,7 +64,7 @@ def test_javascript_long_window_filters_and_point_budget(tmp_path): import shutil import subprocess - from verdi.analysis.html_report import TEMPLATE + from verdi_agents.analysis.html_report import TEMPLATE node = shutil.which('node') if not node: diff --git a/tests/analysis/test_metrics.py b/tests/analysis/test_metrics.py index e9d5383..6b2d960 100644 --- a/tests/analysis/test_metrics.py +++ b/tests/analysis/test_metrics.py @@ -1,10 +1,10 @@ -"""Metric definition tests (P1-E).""" +"""Tests for the metric definitions.""" from __future__ import annotations import pytest from tests.analysis.conftest import appeal, event, probe, sanction, world_state -from verdi.analysis.metrics import ( +from verdi_agents.analysis.metrics import ( attempted_vs_executed_violations, false_sanction_rate, gini, diff --git a/tests/analysis/test_outcome_metrics.py b/tests/analysis/test_outcome_metrics.py index 527da5a..8cf7008 100644 --- a/tests/analysis/test_outcome_metrics.py +++ b/tests/analysis/test_outcome_metrics.py @@ -1,4 +1,4 @@ -"""Offline tests for P0-D primary outcome metrics + comparative report. +"""Offline tests for the primary outcome metrics and the comparative report. Runs REAL offline paired campaigns (scripted fake caller, small ticks — same pattern as tests/experiments/test_paired_run.py) into tmp_path and @@ -17,9 +17,9 @@ import pytest import yaml -from verdi.agents.client import ModelResult -from verdi.analysis.comparative_report import write_comparison_html -from verdi.analysis.outcome_metrics import ( +from verdi_agents.agents.client import ModelResult +from verdi_agents.analysis.comparative_report import write_comparison_html +from verdi_agents.analysis.outcome_metrics import ( RunArtifactError, completion_rate, exclusion_episodes, @@ -31,7 +31,7 @@ write_metrics_csv, write_metrics_json, ) -from verdi.experiments.paired_run import load_paired_config, run_pair +from verdi_agents.experiments.paired_run import load_paired_config, run_pair AGENTS = ("alice", "bob", "chen", "dave") @@ -393,7 +393,7 @@ def test_exclusion_repeat_episode_flag_synthetic(tmp_path): def test_exclusion_blocks_transfer_deposit_in_ledger(pair_root): """While dave is under sanction his deposit attempts are rejected with law:under_sanction — the metric's 'excluded from' definition is real.""" - from verdi.analysis.outcome_metrics import _read_jsonl + from verdi_agents.analysis.outcome_metrics import _read_jsonl events = _read_jsonl(_arm_dir(pair_root, "fixed"), "events") under_sanction_rejects = [ @@ -609,7 +609,7 @@ def test_missing_run_dir_raises_friendly_error(tmp_path): def test_main_cli_writes_three_files(tmp_path, pair_root): - from verdi.analysis import outcome_metrics + from verdi_agents.analysis import outcome_metrics exit_code = outcome_metrics.main(["--pair-root", str(pair_root)]) assert exit_code == 0 diff --git a/tests/analysis/test_report.py b/tests/analysis/test_report.py index 8480afc..4b4be03 100644 --- a/tests/analysis/test_report.py +++ b/tests/analysis/test_report.py @@ -1,4 +1,4 @@ -"""RunReport serialization tests (P1-E).""" +"""Tests for RunReport serialization and the paired-arms statistical report.""" from __future__ import annotations import json @@ -7,27 +7,27 @@ import pytest -from verdi.analysis import RunReport, RunReportError -from verdi.analysis.estimates import ESTIMANDS, paired_estimates, run_estimand_values -from verdi.analysis.report import ( +from verdi_agents.analysis import RunReport, RunReportError +from verdi_agents.analysis.estimates import ESTIMANDS, paired_estimates, run_estimand_values +from verdi_agents.analysis.report import ( G5_ESTIMANDS, G5_LIMITATIONS, build_g5_report, write_g5_report, ) -from verdi.experiments.treatments import ( +from verdi_agents.experiments.treatments import ( placebo_treatment, rights_restoration_treatment, ) -from verdi.pilot.driver import PilotDriver, SeededScheduler, _precompute_dispatch -from verdi.schemas import ExperimentManifest -from verdi.world.branch import ( +from verdi_agents.reference_run.driver import PilotDriver, SeededScheduler, _precompute_dispatch +from verdi_agents.schemas import ExperimentManifest +from verdi_agents.world.branch import ( BranchSpec, branch_from_snapshot, manifest_for_branch, ) -from verdi.world.reducer import WorldReducer -from verdi.world.replay import replay_in_memory +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.replay import replay_in_memory def _manifest() -> ExperimentManifest: @@ -105,13 +105,13 @@ def test_roundtrip_via_jsonl_parse(): # --------------------------------------------------------------------------- # -# G5 statistical sanity report (P2 WS-4): pilot + WS-2 arms end-to-end +# Paired-arms statistical report: reference run + treatment/placebo arms end-to-end # --------------------------------------------------------------------------- # -# SF4: derive the repo root from THIS test file's location — the -# checkout (worktree or main) that contains src/verdi/pilot/driver.py — +# Derive the repo root from THIS test file's location — the +# checkout (worktree or main) that contains src/verdi_agents/reference_run/driver.py — # instead of a hardcoded absolute path. Optional override via the -# VERDI_REPO_ROOT env var: use it only when the pilot's recorded assets +# VERDI_REPO_ROOT env var: use it only when the reference run's recorded assets # live in a different checkout than the code under test. # parents: [0]=analysis, [1]=tests, [2]=checkout root G5_REPO_ROOT = Path(os.environ.get("VERDI_REPO_ROOT") or @@ -119,21 +119,21 @@ def test_roundtrip_via_jsonl_parse(): def _pilot(): - """Run the pilot with the derived repo root. + """Run the reference run with the derived repo root. ``PilotDriver`` itself would already default to - ``Path(__file__).parents[3]`` of ``src/verdi/pilot/driver.py``; we + ``Path(__file__).parents[3]`` of ``src/verdi_agents/reference_run/driver.py``; we pass the explicitly-derived root so the derivation lives in one, testable place and an env-var override can redirect it. """ - assert (G5_REPO_ROOT / "src" / "verdi" / "pilot" / "driver.py").is_file(), ( - f"G5_REPO_ROOT {G5_REPO_ROOT} does not contain src/verdi/pilot/driver.py" + assert (G5_REPO_ROOT / "src" / "verdi_agents" / "reference_run" / "driver.py").is_file(), ( + f"G5_REPO_ROOT {G5_REPO_ROOT} does not contain src/verdi_agents/reference_run/driver.py" ) return PilotDriver(G5_REPO_ROOT).run() def _g5_setup(): - """Run the pilot and fork both WS-2 arms from the same branch point. + """Run the reference run and fork both arms (treatment, placebo) from the same branch point. Returns (pilot, snapshot, prefix_log, remaining_tape, placebo_result, treatment_result, dispatch_ticks). @@ -175,7 +175,7 @@ def _g5_setup(): treatment_spec, ) # both arms are inert on pilot_v1 (appeal dismissed, no sealed - # restoration) and must fold to the pilot's final hash exactly. + # restoration) and must fold to the reference run's final hash exactly. assert placebo_result.branch_state_hash == pilot.final_state.state_hash assert treatment_result.branch_state_hash == pilot.final_state.state_hash # the placeholder prefix_log is the same for both arms @@ -184,7 +184,7 @@ def _g5_setup(): def _arm_states_by_tick(remaining, snapshot, dispatch_ticks, snapshot_ticks): - """Per-LIFECYCLE-tick snapshots for one branch fold (ADR-008). + """Per-LIFECYCLE-tick snapshots for one branch fold. The fold starts at the first governance step (tick 10 in pilot_v1): the first ``n_prefix`` dispatch entries (ticks < 10) were consumed by @@ -192,9 +192,9 @@ def _arm_states_by_tick(remaining, snapshot, dispatch_ticks, snapshot_ticks): order to ``dispatch_ticks[n_prefix:]``; governance steps carry their lifecycle tick in ``payload['tick']``. Snapshots are recorded at the cadence given by ``snapshot_ticks`` plus the terminal tick, exactly - mirroring the pilot's keying. ``snapshot_ticks`` is derived from the - pilot run's own recorded ``state_snapshots`` keys (SF4), not a - frozen tuple. + mirroring the reference run's keying. ``snapshot_ticks`` is derived + from the reference run's own recorded ``state_snapshots`` keys, not a + hardcoded tuple. Returns ``(states_by_tick, final_state)``. """ @@ -249,9 +249,9 @@ def test_g5_report_pilot_v1(tmp_path): placebo_result, treatment_result, dispatch_ticks, ) = _g5_setup() - # ---- snapshot cadence: the pilot's own recorded keys (SF4) -------------- + # ---- snapshot cadence: the reference run's own recorded keys ----------- # Exclude the initial t0 snapshot: the branch fold starts at the - # first governance step, so the arm cadence is the pilot's cadence + # first governance step, so the arm cadence is the reference run's cadence # ticks only (its t0 predates the branch point). snapshot_ticks = tuple(t for t, _ in pilot.state_snapshots if t > 0) assert snapshot_ticks, "pilot recorded no post-initial snapshots" @@ -286,7 +286,7 @@ def test_g5_report_pilot_v1(tmp_path): assert placebo_run["manifest"]["branch_origin"] == "counterfactual-v2" assert treatment_run["manifest"]["treatment_seed"] == 1 - # ---- G5: arms are inert by construction -------------------------------- + # ---- arms are inert by construction ------------------------------------ assert ( placebo_run["estimates"] == treatment_run["estimates"] == {e: float(v) for e, v in placebo_run["estimates"].items()} @@ -321,7 +321,7 @@ def test_g5_report_pilot_v1(tmp_path): assert p1.read_bytes() == p2.read_bytes() # deterministic bytes assert json.loads(p1.read_text(encoding="utf-8")) is not None - # ---- G5 report content checks -------------------------------------------- + # ---- report content checks ----------------------------------------------- for arm in ("pilot", "treatment", "placebo"): traj = report1["trajectories"][arm] assert set(traj["estimates"]) == set(ESTIMANDS) @@ -341,7 +341,7 @@ def test_g5_report_pilot_v1(tmp_path): assert "n_pairs < 10" in joined assert "recorded tape" in joined assert set(report1["limitations"]) == set(G5_LIMITATIONS) - # SF3b: the null-control CI leg is computed over the ACTUAL + # The null-control CI leg is computed over the ACTUAL # per-estimand differences (here all zero), never constant zeros as # an input label. assert report1["null_control"]["ci_input"] == "per-estimand differences" @@ -349,11 +349,11 @@ def test_g5_report_pilot_v1(tmp_path): # --------------------------------------------------------------------------- # -# SF3b: null control is computed over actual differences, not constant zeros +# Null control is computed over actual differences, not constant zeros # --------------------------------------------------------------------------- # def _single_result(estimate): - from verdi.analysis.statistics import PairedResult + from verdi_agents.analysis.statistics import PairedResult return PairedResult( estimate=estimate, ci_low=None, ci_high=None, n_pairs=1, sufficient=False @@ -361,12 +361,12 @@ def _single_result(estimate): def test_null_control_nonzero_differences_fail_ci_leg(): - # SF3b: every per-estimand difference is a constant 5.0 (non-zero). + # Every per-estimand difference is a constant 5.0 (non-zero). # The CI must be computed over these REAL differences, not constant # zeros: the bootstrap CI of the mean over [5.0]*6 is [5, 5], which # does NOT contain 0, so the CI leg must FAIL. The old code computed # over [0,0,0,0] and would have reported passed=True here. - from verdi.analysis.report import _null_control + from verdi_agents.analysis.report import _null_control pairs = {name: _single_result(5.0) for name in G5_ESTIMANDS} control = _null_control(pairs) @@ -380,9 +380,9 @@ def test_null_control_nonzero_differences_fail_ci_leg(): def test_null_control_zero_differences_ci_is_over_real_zeros(): - # SF3b: all-zero differences -> CI trivially [0,0], contains 0, + # All-zero differences -> CI trivially [0,0], contains 0, # passes truthfully (this is the inert-arm wiring check). - from verdi.analysis.report import _null_control + from verdi_agents.analysis.report import _null_control pairs = {name: _single_result(0.0) for name in G5_ESTIMANDS} control = _null_control(pairs) @@ -393,10 +393,10 @@ def test_null_control_zero_differences_ci_is_over_real_zeros(): def test_null_control_insufficient_pairs_marks_null(): - # SF3b: fewer than two difference values -> CI cannot be computed; + # Fewer than two difference values -> CI cannot be computed; # the field is null with an explicit marker, not computed over # constant zeros. - from verdi.analysis.report import _null_control + from verdi_agents.analysis.report import _null_control pairs = {G5_ESTIMANDS[0]: _single_result(0.0)} control = _null_control(pairs) @@ -407,7 +407,7 @@ def test_null_control_insufficient_pairs_marks_null(): # --------------------------------------------------------------------------- # -# SF1b: build_g5_report validates the required run-dict key set up front +# build_g5_report validates the required run-dict key set up front # --------------------------------------------------------------------------- # def _valid_run_dict() -> dict: @@ -447,7 +447,7 @@ class _Pilot: def test_build_g5_report_missing_key_raises(): - # SF1b: a missing required key raises ValueError naming the key, + # A missing required key raises ValueError naming the key, # instead of silently writing a null into the statistical report. run = _valid_run_dict() del run["final_state_hash"] @@ -459,7 +459,7 @@ def test_build_g5_report_missing_key_raises(): def test_build_g5_report_missing_estimates_and_raw_raises(): - # SF1b: neither 'estimates' nor both 'states_by_tick'+'log' present. + # Neither 'estimates' nor both 'states_by_tick'+'log' present. run = _valid_run_dict() del run["estimates"] with pytest.raises(ValueError, match="estimates"): @@ -470,7 +470,7 @@ def test_build_g5_report_missing_estimates_and_raw_raises(): def test_build_g5_report_raw_source_satisfies_estimates(): - # SF1b: 'states_by_tick' + 'log' in place of 'estimates' is accepted. + # 'states_by_tick' + 'log' in place of 'estimates' is accepted. from tests.analysis.conftest import world_state run = _valid_run_dict() @@ -491,13 +491,13 @@ def test_build_g5_report_raw_source_satisfies_estimates(): # --------------------------------------------------------------------------- # -# SF4: repo root derivation +# Repo root derivation # --------------------------------------------------------------------------- # def test_repo_root_derivation_contains_driver(): - # SF4: the root is derived from this test file's own location, not a - # frozen hardcoded absolute path — so it points at the checkout - # containing src/verdi/pilot/driver.py and the pilot assets. + # The root is derived from this test file's own location, not a + # hardcoded absolute path, so it points at the checkout containing + # src/verdi_agents/reference_run/driver.py and the reference-run assets. assert G5_REPO_ROOT.is_absolute() # derived from __file__ (or the VERDI_REPO_ROOT override), not frozen expected = ( @@ -506,5 +506,5 @@ def test_repo_root_derivation_contains_driver(): else Path(__file__).resolve().parents[2] ) assert expected.resolve() == G5_REPO_ROOT - assert (G5_REPO_ROOT / "src" / "verdi" / "pilot" / "driver.py").is_file() - assert (G5_REPO_ROOT / "experiments" / "scenarios" / "pilot_v1.json").is_file() + assert (G5_REPO_ROOT / "src" / "verdi_agents" / "reference_run" / "driver.py").is_file() + assert (G5_REPO_ROOT / "tests" / "fixtures" / "reference_run" / "scenarios" / "pilot_v1.json").is_file() diff --git a/tests/analysis/test_scenario.py b/tests/analysis/test_scenario.py index 4a48d7f..c50d915 100644 --- a/tests/analysis/test_scenario.py +++ b/tests/analysis/test_scenario.py @@ -6,8 +6,13 @@ import pytest from tests.analysis.conftest import SCENARIO, TAPE -from verdi.analysis import LoadedScenario, ScenarioLoader, ScenarioLoadError, canonical_tape_hash -from verdi.schemas import ExperimentManifest +from verdi_agents.analysis import ( + LoadedScenario, + ScenarioLoader, + ScenarioLoadError, + canonical_tape_hash, +) +from verdi_agents.schemas import ExperimentManifest def _stored_hash() -> str: diff --git a/tests/analysis/test_statistics.py b/tests/analysis/test_statistics.py index 4c368cd..93ebedc 100644 --- a/tests/analysis/test_statistics.py +++ b/tests/analysis/test_statistics.py @@ -1,4 +1,4 @@ -"""Tests for verdi.analysis.statistics (P2 WS-3, G5 statistical sanity). +"""Tests for verdi_agents.analysis.statistics (paired estimates and bootstrap CIs). All tests are deterministic: fixed seeds and fixed synthetic data only. No LLM, no network, stdlib only. @@ -10,7 +10,7 @@ import pytest -from verdi.analysis.statistics import ( +from verdi_agents.analysis.statistics import ( PairedResult, bootstrap_ci, null_control_passes, @@ -38,7 +38,7 @@ def test_bootstrap_ci_different_seed_may_differ(): def test_bootstrap_ci_percentiles_match_statistics_quantiles(): - """P2 review F4: the CI endpoints must match the stdlib + """The CI endpoints must match the stdlib ``statistics.quantiles(means, n=200, method="inclusive")`` convention to 1 ulp (abs diff <= 1e-12 * max(1, |value|)). @@ -204,18 +204,18 @@ def test_bootstrap_coverage_sanity_true_shift_one(): def test_module_imports_are_stdlib_only(): - """Guard ADR-009: WS-3 must not import numpy/scipy or verdi internals.""" - import verdi.analysis.statistics as mod + """The statistics module must not import numpy/scipy or verdi internals.""" + import verdi_agents.analysis.statistics as mod for value in vars(mod).values(): module = str(getattr(value, "__module__", "")) assert not module.startswith("numpy"), "numpy referenced" assert not module.startswith("scipy"), "scipy referenced" - # The module's own import statements are stdlib-only (ADR-009). + # The module's own import statements are stdlib-only. import inspect source = inspect.getsource(mod) assert "import numpy" not in source assert "import scipy" not in source - assert "verdi.experiments" not in source + assert "verdi_agents.experiments" not in source assert math.isfinite(bootstrap_ci([1.0, 2.0], seed=0)[0]) diff --git a/tests/audit/test_adapter.py b/tests/audit/test_adapter.py index ec40295..e70f783 100644 --- a/tests/audit/test_adapter.py +++ b/tests/audit/test_adapter.py @@ -1,10 +1,10 @@ -"""G0/G1 tests for the audit adapter (P1-D).""" +"""Tests for the audit adapter.""" from __future__ import annotations import pytest -from verdi.audit import AuditAdapter, InMemoryTraceSource -from verdi.schemas import AuditRecord, WorldEvent +from verdi_agents.audit import AuditAdapter, InMemoryTraceSource +from verdi_agents.schemas import AuditRecord, WorldEvent RUBRIC = "rubric-1" SCORER = "option-rubric/v1.0" @@ -129,6 +129,6 @@ def test_inmemory_fetch_is_isolated_per_trace_and_stable() -> None: def test_trace_source_protocol_runtime_checkable() -> None: - from verdi.audit.adapter import TraceSource + from verdi_agents.audit.adapter import TraceSource assert isinstance(InMemoryTraceSource(), TraceSource) diff --git a/tests/audit/test_audit_regressions.py b/tests/audit/test_audit_regressions.py index ace0ca8..ea8ef10 100644 --- a/tests/audit/test_audit_regressions.py +++ b/tests/audit/test_audit_regressions.py @@ -1,4 +1,4 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import asyncio @@ -6,8 +6,8 @@ import pytest -import verdi.audit.interview_audit as ia -from verdi.audit.judge_live import _assert_no_key_material +import verdi_agents.audit.interview_audit as ia +from verdi_agents.audit.judge import _assert_no_key_material @pytest.mark.parametrize("text", [ diff --git a/tests/audit/test_judge_live.py b/tests/audit/test_judge.py similarity index 97% rename from tests/audit/test_judge_live.py rename to tests/audit/test_judge.py index b053560..976062d 100644 --- a/tests/audit/test_judge_live.py +++ b/tests/audit/test_judge.py @@ -1,4 +1,4 @@ -"""Offline tests for the ADR-012 live judge pass (synthetic tape + ledger). +"""Offline tests for the live judge pass (synthetic tape + ledger). Fully offline: no network, no real sleeps. The judge model is faked with a recorded ``acompletion`` double (including one unparseable response and one @@ -20,8 +20,8 @@ import pytest -import verdi.audit.judge_live as jl -from verdi.audit.judge_live import ( +import verdi_agents.audit.judge as jl +from verdi_agents.audit.judge import ( JUDGE_SYSTEM_PROMPT, SAMPLE_SEED, JudgeTapeCapture, @@ -35,19 +35,19 @@ wilson_ci_95, write_judge_report, ) -from verdi.audit.live_bridge import ( +from verdi_agents.audit.trace_audit import ( LiveLedgerError, correlate_tape_to_ledger, load_ledger_events, ) -from verdi.schemas import WorldEvent +from verdi_agents.schemas import WorldEvent FAKE_KEY = "sk-abc1234567890testkey" FAKE_URL = "https://judge.example.invalid/api/v1" MODEL = "qwen-fake-27b" # --------------------------------------------------------------------------- -# Synthetic fixtures (same shape as the real L1 tape / ledger) +# Synthetic fixtures (same shape as the real reference-run tape / ledger) # --------------------------------------------------------------------------- @@ -127,7 +127,7 @@ def _write(path: Path, rows: list[dict]) -> None: @pytest.fixture() def judge_tape_ledger(tmp_path): """6 dispatches; the sole engine DISPATCH rejection is event 42 (tick 11) — - mirroring the L1 run's shape in miniature.""" + mirroring the original live run's shape in miniature.""" events = [ _event(0, actor_id="a-x", task_id="task0", kind="task_assigned", decision="rejected"), _event(2, actor_id="a-1", task_id="t1", tick=1), @@ -531,7 +531,7 @@ def test_report_disagreement_enumerated_and_rates( # A1 m2: the agreement metric definition is in the report. assert report["agreement_metric"].startswith("agreement = (judge_stance == engine_stance)") assert any("AGREEMENT METRIC DEFINITION" in x for x in report["limitations"]) - # A1 M2: overall + per-class CIs with counts; n=1 class flagged. + # Overall + per-class CIs with counts; n=1 class flagged. assert s["agreement_wilson_ci_95"] == [v for v in wilson_ci_95(4, 6)] assert s["accepted_breakdown"]["rows"] == 5 assert s["accepted_breakdown"]["agreements"] == 4 @@ -709,7 +709,7 @@ def test_report_function_takes_only_plain_data() -> None: assert a["summary"]["disagreement_count"] == 1 d = a["summary"]["disagreements"][0] assert d["event_id"] == 42 - # A1-2 M1: honest reason derived from which component diverged — + # Honest reason derived from which component diverged — # here BOTH the model judge and the code judge diverge from the # engine, so it must NOT claim a single-model-only disagreement. assert "model-judge (yes)" in d["reason"] @@ -780,11 +780,11 @@ async def acompletion(self, **kwargs): # --------------------------------------------------------------------------- -# (5) real-artifact guard: L1 tape/ledger -> all-81 sample incl. event 42 +# (5) real-artifact guard: original live run's tape/ledger -> all-81 sample incl. event 42 # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- -# (6) A1-2 M2: Wilson CI math (stdlib-only) validated against known values +# (6) Wilson CI math (stdlib-only) validated against known values # --------------------------------------------------------------------------- @@ -794,7 +794,7 @@ def test_wilson_ci_known_values() -> None: lo, hi = wilson_ci_95(10, 100) assert lo == pytest.approx(0.055229, abs=1e-4) assert hi == pytest.approx(0.174366, abs=1e-4) - # n=1 classes (the L1 rejection class shape): CI spans (0,1). + # n=1 classes (the original run's rejection class shape): CI spans (0,1). assert wilson_ci_95(1, 1) == (pytest.approx(0.206549, abs=1e-4), 1.0) assert wilson_ci_95(0, 1) == (0.0, pytest.approx(0.793451, abs=1e-4)) # Extreme cases: the interval is within float eps of the clipped @@ -843,7 +843,7 @@ def test_agreement_ci_excludes_none_entries() -> None: # --------------------------------------------------------------------------- -# (7) A1-2 M3: judge-path ORACLE fixture (offline; known-correct rows) +# (7) judge-path ORACLE fixture (offline; known-correct rows) # The judge CODE PATH (build_judge_rows -> parse_judge_verdict -> # report) must score each known-correct adjudication row correctly # with a fake judge returning FIXED verdicts. No network. @@ -864,7 +864,7 @@ def test_judge_path_oracle_fixture(judge_tape_ledger, tmp_path, monkeypatch) -> engine, but the code judge is STRUCTURALLY "yes" (accepted-task closed-form rule), so agreement (judge_vs_engine AND code_correct) is False with an honest structural-artifact - reason — exactly the A1-2 M1 scenario; + reason — exactly the divergence-reason scenario; 3. an UNPARSEABLE judge response -> quarantined, agreement None. Every other row judges in agreement (yes), so the oracle expects exactly these 3 rows to deviate and nothing else. @@ -892,7 +892,7 @@ def test_judge_path_oracle_fixture(judge_tape_ledger, tmp_path, monkeypatch) -> events = load_ledger_events(ledger_path) sample = build_judge_sample(tape_path, ledger_path) code_rows = build_judge_rows(sample, tape, events) - from verdi.audit.live_bridge import correlate_tape_to_ledger, report_correlation + from verdi_agents.audit.trace_audit import correlate_tape_to_ledger, report_correlation corr = report_correlation(tape, events, correlate_tape_to_ledger(tape, events)) report = build_judge_report(sample, code_rows, run["verdicts"], run["meta"], corr) by_event = {r["event_id"]: r for r in report["rows"]} @@ -902,7 +902,7 @@ def test_judge_path_oracle_fixture(judge_tape_ledger, tmp_path, monkeypatch) -> # 2) violating row judged no: judge agrees with the engine # (judge_vs_engine True), but the code judge's closed-form # accepted-task rule says "yes" -> agreement is False and the - # reason is the STRUCTURAL-artifact wording (A1-2 M1), never a + # reason is the STRUCTURAL-artifact wording, never a # claim of a divergent reading of the same evidence. assert by_event[42]["judge_verdict"]["stance"] == "no" assert by_event[42]["judge_vs_engine"] is True @@ -932,7 +932,7 @@ def test_judge_path_oracle_fixture(judge_tape_ledger, tmp_path, monkeypatch) -> # --------------------------------------------------------------------------- -# (8) A1-2 minors: budget-at-retry and budget-at-quarantine branches +# (8) budget-at-retry and budget-at-quarantine branches # --------------------------------------------------------------------------- @@ -1026,7 +1026,7 @@ def test_quarantined_retry_blocked_at_budget_cap( def test_fatal_error_writes_partial_tape_and_meta( judge_tape_ledger, tmp_path, monkeypatch ) -> None: - """A1-2 Eng M1: a FATAL, unexpected failure OUTSIDE the per-attempt + """A FATAL, unexpected failure OUTSIDE the per-attempt guard (here: the verdict bookkeeping itself) must persist the rows judged so far (partial tape + meta, flagged) before re-raising — no silent total loss. Per-attempt judge failures are audited as @@ -1078,7 +1078,7 @@ def _fatal_parse(content: str) -> dict: # --------------------------------------------------------------------------- -# (9) A1-2 Eng M2: REAL SimpleAudit ModelAuditor path, offline +# (9) REAL SimpleAudit ModelAuditor path, offline # (mock transport only; NO _build_auditor spy monkeypatch) # --------------------------------------------------------------------------- @@ -1220,12 +1220,12 @@ async def _go(): assert parse_judge_verdict(by_event_content[5])["quarantined"] is True -L1_TAPE = Path("/Users/sushantgautam/Documents/verdi/runs/live/20261010T071750Z/live_tape.jsonl") +L1_TAPE = Path(__file__).resolve().parents[2] / "runs" / "live" / "20261010T071750Z" / "live_tape.jsonl" L1_LEDGER = L1_TAPE.parent / "event_log.jsonl" def test_l1_sample_all_81_with_event_42() -> None: - """Guard over the real captured L1 artifacts (skipped when absent).""" + """Guard over the real captured live-run artifacts (skipped when absent).""" if not (L1_TAPE.is_file() and L1_LEDGER.is_file()): pytest.skip("captured L1 artifacts not present; guard test skipped") sample = build_judge_sample(L1_TAPE, L1_LEDGER) diff --git a/tests/audit/test_probing_target.py b/tests/audit/test_probing_target.py index 4ba24bf..68239b7 100644 --- a/tests/audit/test_probing_target.py +++ b/tests/audit/test_probing_target.py @@ -13,7 +13,7 @@ pytest.importorskip("simpleaudit") -from verdi.audit.interview_audit import ProbingInterviewTarget # noqa: E402 +from verdi_agents.audit.interview_audit import ProbingInterviewTarget # noqa: E402 class _FakeClient: @@ -69,7 +69,7 @@ async def generate_structured(self, *a, **k): # --------------------------------------------------------------------------- # # _as_int_1_10 coercion (judge 1-10 score parsing) # --------------------------------------------------------------------------- # -from verdi.audit.interview_audit import _as_int_1_10 # noqa: E402 +from verdi_agents.audit.interview_audit import _as_int_1_10 # noqa: E402 def test_as_int_1_10_accepts_valid_forms(): diff --git a/tests/audit/test_simpleaudit_bridge.py b/tests/audit/test_simpleaudit_bridge.py index 9275bef..8f27ae1 100644 --- a/tests/audit/test_simpleaudit_bridge.py +++ b/tests/audit/test_simpleaudit_bridge.py @@ -5,8 +5,8 @@ import pytest -import verdi.audit.simpleaudit_bridge as bridge -from verdi.audit.simpleaudit_bridge import ( +import verdi_agents.audit.simpleaudit_bridge as bridge +from verdi_agents.audit.simpleaudit_bridge import ( SimpleAuditNotInstalledError, SimpleAuditTraceSource, build_simpleaudit_trace_source, @@ -27,7 +27,7 @@ def test_module_import_does_not_import_simpleaudit() -> None: # has no side effect: importing it must never import simpleaudit. import importlib - fresh_name = "verdi.audit.simpleaudit_bridge" + fresh_name = "verdi_agents.audit.simpleaudit_bridge" original = sys.modules.get(fresh_name) sys.modules.pop(fresh_name, None) sys.modules.pop("simpleaudit", None) diff --git a/tests/audit/test_live_bridge.py b/tests/audit/test_trace_audit.py similarity index 95% rename from tests/audit/test_live_bridge.py rename to tests/audit/test_trace_audit.py index 5c615fe..6e82439 100644 --- a/tests/audit/test_live_bridge.py +++ b/tests/audit/test_trace_audit.py @@ -1,4 +1,4 @@ -"""Offline tests for the L2 G4 live audit bridge (synthetic tape + ledger). +"""Offline tests for the live trace audit (synthetic tape + ledger). Fully offline: no network, no model calls. Covers the correlation rule (sealed event_id, never positional), orphan/missing reporting, byte-stable @@ -11,14 +11,14 @@ import pytest -from verdi.audit.live_bridge import ( +from verdi_agents.audit.trace_audit import ( LiveLedgerError, LiveTapeTraceSource, audit_live_run, correlate_tape_to_ledger, select_judge_sample, ) -from verdi.schemas import WorldEvent +from verdi_agents.schemas import WorldEvent # --------------------------------------------------------------------------- # Synthetic fixtures @@ -161,7 +161,7 @@ def test_source_fetch_uses_sealed_span_key(mini_tape) -> None: def test_orphan_span_reported_not_dropped(tmp_path) -> None: """A span whose sealed event id is absent from the ledger is reported.""" - from verdi.audit.live_bridge import report_correlation + from verdi_agents.audit.trace_audit import report_correlation events = [ _event(2, actor_id="a-1", task_id="t1", tick=1), @@ -292,22 +292,22 @@ def test_judge_step_uses_simpleaudit_and_records_path(mini_tape) -> None: assert per_call[2]["model"] == "synthetic-1" assert per_call[2]["judge_score"] == 1.0 assert per_call[5]["judge_score"] == 1.0 - # Limitations name the known pilot gap. + # Limitations name the known reference-run gap. assert any("RunReport" in s for s in report["limitations"]) # --------------------------------------------------------------------------- -# (f) guard over the REAL captured pilot tape + ledger (skip if absent) +# (f) guard over the REAL captured reference-run tape + ledger (skip if absent) # --------------------------------------------------------------------------- -PILOT_TAPE = Path("/Users/sushantgautam/Documents/verdi/runs/live/20261010T071750Z/live_tape.jsonl") +PILOT_TAPE = Path(__file__).resolve().parents[2] / "runs" / "live" / "20261010T071750Z" / "live_tape.jsonl" def _resolve_pilot_ledger() -> Path: """Ledger provenance: stable run dir first (regenerated byte-identical via - run_pilot_from_live_tape), then the original L1 worktree path if present.""" + run_pilot_from_live_tape), then the original live-run worktree path if present.""" stable = PILOT_TAPE.parent / "event_log.jsonl" - legacy = Path("/Users/sushantgautam/Documents/verdi-L1/runs/live/event_log.jsonl") + legacy = Path(__file__).resolve().parents[3] / "verdi-L1" / "runs" / "live" / "event_log.jsonl" if stable.is_file(): return stable return legacy @@ -317,7 +317,7 @@ def _resolve_pilot_ledger() -> Path: def test_pilot_tape_ledger_actor_task_unique_and_1_to_1() -> None: - """Guard: pilot_v1's (actor_id, task_id) pairs are unique on both sides + """Guard: the reference scenario's (actor_id, task_id) pairs are unique on both sides and map 1:1 — the invariant the sealed-event_id correlation relies on. Runs over the real captured artifacts; skipped (with a clear message) diff --git a/tests/audit/test_g4_offline.py b/tests/audit/test_trace_audit_offline.py similarity index 92% rename from tests/audit/test_g4_offline.py rename to tests/audit/test_trace_audit_offline.py index 8222ff4..6d201a5 100644 --- a/tests/audit/test_g4_offline.py +++ b/tests/audit/test_trace_audit_offline.py @@ -1,17 +1,17 @@ -"""G4 offline remainder (P2 WS-5). +"""Offline checks for the trace audit that need no SimpleAudit or live model. -Proves the parts of gate G4 (ACCEPTANCE.md) that are verifiable WITHOUT +Proves the parts of the trace-audit check that are verifiable WITHOUT SimpleAudit installed and WITHOUT a live model endpoint: 1. Absence path: with SimpleAudit absent, the bridge degrades to the documented ``SimpleAuditNotInstalledError`` (the offline blocker). -2. Adapter contract: spans carry ``name`` (NOT ``span_name`` — LESSONS.md - known bug), and audit correlation is by sealed ``event_id``, never by +2. Adapter contract: spans carry ``name`` (NOT ``span_name`` — a + past bug), and audit correlation is by sealed ``event_id``, never by positional index. 3. Judge/scorer contract (offline rubric path only, no LLM): match vs divergence ordering, frozen score range, input immutability, determinism — all checked against a local reference. -4. G4 residual: the recorded pilot's RunReport carries the audit/judge +4. Remaining gap: the recorded reference run's RunReport carries the audit/judge fields the gate requires, and its limitations already mark live verification as not measured (see the PENDING/OFFLINE assertion for the integration note on making that explicit). @@ -25,14 +25,14 @@ import pytest from pydantic_core import ValidationError -from verdi.audit import AuditAdapter, InMemoryTraceSource -from verdi.audit.simpleaudit_bridge import ( +from verdi_agents.audit import AuditAdapter, InMemoryTraceSource +from verdi_agents.audit.simpleaudit_bridge import ( SimpleAuditNotInstalledError, build_simpleaudit_trace_source, ) -from verdi.measurement.scoring import JudgeProtocol, RubricScorer, score_probe -from verdi.pilot.driver import run_pilot -from verdi.schemas import AuditRecord, ProbeItem, WorldEvent +from verdi_agents.measurement.scoring import JudgeProtocol, RubricScorer, score_probe +from verdi_agents.reference_run.driver import run_pilot +from verdi_agents.schemas import AuditRecord, ProbeItem, WorldEvent RUBRIC_ID = "rubric-g4-offline" SCORER_ID = f"{RUBRIC_ID}/v1.0" @@ -58,15 +58,15 @@ def test_simpleaudit_absence_degrades_to_installed_error() -> None: # It is an ImportError subclass so generic import handling still works. assert isinstance(excinfo.value, ImportError) message = str(excinfo.value) - # Names the missing package and states installation is required for live G4. + # Names the missing package and states installation is required for a live audit. assert "simpleaudit" in message.lower() assert "not installed" in message - # Actionable remediation for the live-G4 path. + # Actionable remediation for the live audit path. assert "PYTHONPATH" in message or "VERDI_SIMPLEAUDIT_PATH" in message # -------------------------------------------------------------------------- # -# 2. Adapter contract (offline, InMemoryTraceSource as in the pilot) +# 2. Adapter contract (offline, InMemoryTraceSource as in the reference run) # -------------------------------------------------------------------------- # def _span(span_id: str, trace_id: str, name: str, agent_id: str) -> dict[str, Any]: """Normalized SimpleAudit span shape: key is ``name``, not ``span_name``.""" @@ -106,7 +106,7 @@ def test_spans_emit_name_key_not_span_name_and_adapter_reads_it() -> None: fetched = source.fetch("t-1") # Contract: the normalized span key is "name" — a "span_name" key is - # silently dropped by the adapter (LESSONS.md span-key regression). + # silently dropped by the adapter (a past span-key regression). assert all("name" in s for s in fetched) assert all("span_name" not in s for s in fetched) @@ -278,10 +278,10 @@ def score(self, item: ProbeItem, response: str) -> float: # -------------------------------------------------------------------------- # -# 4. G4 residual documentation +# 4. Remaining-gap documentation # -------------------------------------------------------------------------- # def test_pilot_report_carries_g4_audit_and_judge_fields() -> None: - """RunReport/PilotRun contain the audit + judge fields G4 requires.""" + """RunReport/PilotRun contain the audit + judge fields the trace audit requires.""" pilot = run_pilot() report = pilot.report assert report is not None @@ -314,9 +314,9 @@ def test_pilot_report_carries_g4_audit_and_judge_fields() -> None: assert "value_drift_by_channel" in report.metrics assert report.manifest["probe_bank_version"] - # G4 residual: live verification is explicitly NOT measured. + # Remaining gap: live verification is explicitly NOT measured. # - # INTEGRATION NOTE (orchestrator): the current limitations text says + # NOTE: the current limitations text says # "no live Qwen/vLLM interaction was measured" and the manifest pins # simpleaudit_version=None, which together document the residual, but no # line names the SimpleAudit live verification as PENDING/OFFLINE. If diff --git a/tests/experiments/__init__.py b/tests/experiments/__init__.py index 091be69..53c8db5 100644 --- a/tests/experiments/__init__.py +++ b/tests/experiments/__init__.py @@ -1 +1 @@ -"""P2 WS-2: experiments package tests (regimes, treatments, matched pairs).""" +"""Tests for the experiments package (regimes, treatments, matched pairs).""" diff --git a/tests/experiments/conftest.py b/tests/experiments/conftest.py index 3825163..1ac7fc3 100644 --- a/tests/experiments/conftest.py +++ b/tests/experiments/conftest.py @@ -1,7 +1,7 @@ -"""Shared fixtures for the G5-close tests (offline only, no network, no .env). +"""Shared fixtures for the pair-judge and pair-audit tests (offline, no network, no .env). -The REAL run root constant is resolved against the MAIN repo root ONLY here -in tests (per worker contract); src code never hardcodes it. +The path to the real recorded run is resolved against the main repo root +only here in tests; src code never hardcodes it. """ from __future__ import annotations @@ -11,13 +11,13 @@ import pytest -#: Main checkout (integrator-owned; tests read the gitignored run root only). -MAIN_REPO_ROOT = Path("/Users/sushantgautam/Documents/verdi") +#: Main checkout (tests only read the gitignored run directory from it). +MAIN_REPO_ROOT = Path(__file__).resolve().parents[2] REAL_G5_RUN_ROOT = MAIN_REPO_ROOT / "runs" / "live" / "g5_pairs" / "20261010T162928Z" requires_real_run_root = pytest.mark.skipif( not REAL_G5_RUN_ROOT.is_dir(), - reason=f"real G5 run root {REAL_G5_RUN_ROOT} not present", + reason=f"real paired-arms run root {REAL_G5_RUN_ROOT} not present", ) @@ -99,10 +99,11 @@ def write_jsonl(path: Path, rows: list[dict]) -> None: def make_mini_run_root(root: Path) -> Path: - """2-pair synthetic G5 run root: 3 tape rows per pair (ticks 1, 2, 13). + """2-pair synthetic paired-arms run root: 3 tape rows per pair (ticks 1, 2, 13). - Clean by construction: B1..B4 all pass. Suffix row (tick 13) is - IDENTICAL across pairs (B4). Each pair also carries a + Clean by construction: all four audits (B1..B4) pass. Suffix row + (tick 13) is IDENTICAL across pairs (prompt-invariance audit, B4). + Each pair also carries a ``treatment/event_log.jsonl`` with one accepted action event per row so the engine outcome maps (decision "accepted"). """ diff --git a/tests/experiments/test_audit_regressions.py b/tests/experiments/test_audit_regressions.py index 2b1f595..5fe4316 100644 --- a/tests/experiments/test_audit_regressions.py +++ b/tests/experiments/test_audit_regressions.py @@ -1,4 +1,4 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import threading @@ -6,8 +6,8 @@ import pytest -from verdi.analysis.comparative_report import _institution_rows -from verdi.experiments import sweep as sw +from verdi_agents.analysis.comparative_report import _institution_rows +from verdi_agents.experiments import sweep as sw from .test_sweep import STARTER, _no_real_env # noqa: F401 - autouse fixture @@ -81,8 +81,8 @@ def interrupting(futures): def test_interview_post_pass_audit_returns_summary(tmp_path, monkeypatch): # audit_from_file is synchronous; wrapping its dict in asyncio.run crashed # every post-pass audit with "a coroutine was expected". - import verdi.audit.interview_audit as ia - from verdi.experiments.interview_experiment import _audit_run + import verdi_agents.audit.interview_audit as ia + from verdi_agents.experiments.interviews import _audit_run (tmp_path / "interviews.jsonl").write_text("{}\n") monkeypatch.setattr(ia, "audit_from_file", lambda *a, **k: {"provenance_rows": 1}) @@ -95,7 +95,7 @@ def test_interview_post_pass_audit_returns_summary(tmp_path, monkeypatch): ("Yes.", "yes"), ('"no"', "no"), ("Yes, I would.", "yes"), ]) def test_probe_parser_accepts_punctuated_yes_no(reply, choice): - from verdi.measurement.longitudinal import _parse_choice + from verdi_agents.measurement.longitudinal import _parse_choice assert _parse_choice(reply, ["yes", "no"]) == (choice, False) @@ -106,13 +106,13 @@ def test_probe_parser_accepts_punctuated_yes_no(reply, choice): ("I rate it 7 out of 10", ("0", True)), ]) def test_probe_parser_likert_takes_last_in_range_integer(reply, expected): - from verdi.measurement.longitudinal import _parse_choice + from verdi_agents.measurement.longitudinal import _parse_choice assert _parse_choice(reply, [str(i) for i in range(6)]) == expected def test_paired_run_without_output_root_fails_readably(tmp_path): - from verdi.experiments.paired_run import run_pair + from verdi_agents.experiments.paired_run import run_pair config = {"experiment_id": "x", "base_scenario": STARTER, "arms": ["fixed"], "seeds": [1], "total_ticks_override": None, "output_root": None} @@ -141,13 +141,13 @@ def launcher(run, conc): ("Answer: 3 (scale 0-5)", "3"), ]) def test_probe_parser_likert_ignores_scale_and_denominator(reply, choice): - from verdi.measurement.longitudinal import _parse_choice + from verdi_agents.measurement.longitudinal import _parse_choice assert _parse_choice(reply, [str(i) for i in range(6)]) == (choice, False) def test_paired_run_rerun_sets_previous_attempt_aside(tmp_path, monkeypatch): - import verdi.experiments.paired_run as pr + import verdi_agents.experiments.paired_run as pr calls = [] @@ -169,7 +169,7 @@ async def fake_arm(*, arm, seed, scenario, run_dir, caller_factory, concurrency) def test_causal_guard_ignores_verbatim_model_text(): - from verdi.experiments.g5_close import _without_model_text, causal_word_guard + from verdi_agents.experiments.pair_audit import _without_model_text, causal_word_guard report = {"judge": {"rows": [{"rationale": "The proposal is causally linked.", "raw": "causal causal causal"}]}} @@ -179,7 +179,7 @@ def test_causal_guard_ignores_verbatim_model_text(): def test_nearest_event_returns_latest_event_by_lifecycle_tick(): - from verdi.measurement.interview_values import _nearest_event + from verdi_agents.measurement.interview_values import _nearest_event events = [ {"event_id": 0, "tick": 1, "actor_id": "e", "kind": "a", @@ -200,13 +200,13 @@ def test_nearest_event_returns_latest_event_by_lifecycle_tick(): ("Answer: 4 — 10x more", "4"), ]) def test_probe_parser_likert_answer_leads_over_trailing_reasoning(reply, choice): - from verdi.measurement.longitudinal import _parse_choice + from verdi_agents.measurement.longitudinal import _parse_choice assert _parse_choice(reply, [str(i) for i in range(6)]) == (choice, False) def test_causal_guard_ignores_model_stance_in_error_field(): - from verdi.experiments.g5_close import _without_model_text, causal_word_guard + from verdi_agents.experiments.pair_audit import _without_model_text, causal_word_guard causal_word_guard(_without_model_text( {"rows": [{"error": "invalid stance 'causally_compliant'"}]})) diff --git a/tests/experiments/test_g5_pairs.py b/tests/experiments/test_counterfactual_pairs.py similarity index 88% rename from tests/experiments/test_g5_pairs.py rename to tests/experiments/test_counterfactual_pairs.py index 5da7f08..e88de25 100644 --- a/tests/experiments/test_g5_pairs.py +++ b/tests/experiments/test_counterfactual_pairs.py @@ -1,17 +1,18 @@ -"""ADR-013 G5 paired arms — offline tests (NO NETWORK, NO REAL SLEEPS). +"""Offline tests for matched treatment/placebo pairs (no network, no real sleeps). -Pins the ADR-013 pipeline against the sealed L1 provenance: -* L1 loading / provenance invariants; -* matching invariants (origin hash B3 + sealed prefix tape hash, +Checks the counterfactual-pairs pipeline against the sealed original live run: +* loading the original live run and its provenance invariants; +* matching invariants (origin state hash + sealed prefix tape hash, ``make_matched_pair`` identity); -* the mandatory placebo-inert null control (same-code invariants per - ADR-013 Addendum A1: prefix byte-identical, arms identical except the - adjudication cascade, equal folded finals, unbroken chains); -* arm folds: treatment (upheld + rights restored) vs placebo (verbatim L1 - dismissal), scaffold invariance (identical final state hash), - cross-arm adjudication-kind isolation, placebo byte-identity to L1; +* the mandatory placebo-inert null control (same-code invariants: + prefix byte-identical, arms identical except the adjudication + cascade, equal folded finals, unbroken chains); +* arm folds: treatment (upheld + rights restored) vs placebo (the + original run's verbatim dismissal), scaffold invariance (identical + final state hash), cross-arm adjudication-kind isolation, placebo + byte-identity to the original run; * arm run dicts satisfy ``analysis.report._validate_arm_run_dict``; -* deterministic G5 report byte-stability (build twice, identical bytes); +* deterministic paired report byte-stability (build twice, identical bytes); * no API-key material in any artifact. The REAL live run (10 pairs × 47 Qwen calls) is script-style outside @@ -26,9 +27,9 @@ import pytest -from verdi.analysis.estimates import ESTIMANDS, paired_estimates -from verdi.analysis.report import _validate_arm_run_dict -from verdi.experiments.g5_pairs import ( +from verdi_agents.analysis.estimates import ESTIMANDS, paired_estimates +from verdi_agents.analysis.report import _validate_arm_run_dict +from verdi_agents.experiments.counterfactual_pairs import ( EXPECTED_SUFFIX_CALLS, HARD_CALL_CAP, L1_EVENT_LOG_PATH, @@ -47,15 +48,15 @@ prefix_event_log, run_arm, ) -from verdi.experiments.pairs import make_matched_pair, tape_hash -from verdi.world.reducer import WorldReducer +from verdi_agents.experiments.pairs import make_matched_pair, tape_hash +from verdi_agents.world.reducer import WorldReducer # Repo root of the checkout this test file belongs to (works in the main # checkout and in any worktree; the null control only needs a git repo). WORKTREE = Path(__file__).resolve().parents[2] pytestmark = pytest.mark.skipif( not L1_TAPE_PATH.is_file() or not L1_EVENT_LOG_PATH.is_file(), - reason="L1 live run artifacts not present (run artifacts, not repo assets)", + reason="original live run artifacts not present (run artifacts, not repo assets)", ) @@ -71,14 +72,14 @@ def context(l1): @pytest.fixture(scope="module") def arms(l1): - """One pair folded from L1's own tape (R = L1 suffix): fast, offline.""" + """One pair folded from the original run's own tape (its own suffix): fast, offline.""" tr = run_arm(WORKTREE, l1.tape_rows, "treatment") pr = run_arm(WORKTREE, l1.tape_rows, "placebo") return tr, pr # --------------------------------------------------------------------------- # -# L1 provenance +# Original live run provenance # --------------------------------------------------------------------------- # @@ -89,7 +90,7 @@ def test_l1_provenance_invariants(l1): assert len(l1.tape_rows) == 81 assert len(l1.prefix_tape_rows) == 34 assert len(l1.suffix_tape_rows) == EXPECTED_SUFFIX_CALLS == 47 - # B3(a): origin hash = sealed after_hash of event 49. + # Origin hash = sealed after_hash of event 49. assert l1.origin_state_hash == l1.event_log.events[49].after_hash @@ -105,9 +106,9 @@ def test_prefix_event_log_chain(l1): def test_matching_context_b3(l1, context): - # B3(a): origin hash equals the folded prefix terminal canonical hash. + # Origin hash equals the folded prefix terminal canonical hash. assert context["folded_prefix_terminal_hash"] == context["origin_state_hash"] - # B3(b): origin event count. + # Origin event count. assert context["origin_event_count"] == 50 # tape_hash is over the sealed prefix ledger. assert context["prefix_tape_hash"] == tape_hash(prefix_event_log(l1)) @@ -127,7 +128,7 @@ def test_arm_manifests_and_matched_pair(l1, context): def test_matching_context_fold_is_pure(l1, context): # Re-folding reproduces the identical terminal hash (determinism). - from verdi.experiments.g5_pairs import _fold_prefix_state + from verdi_agents.experiments.counterfactual_pairs import _fold_prefix_state # repo_root=None resolves package-relative (the worktree src layout). h1 = WorldReducer.canonical_state_hash(_fold_prefix_state(l1)) @@ -141,7 +142,7 @@ def test_matching_context_fold_is_pure(l1, context): def test_null_control_placebo_inert(l1): - """Same-code invariants (ADR-013 Addendum A1) all hold on the L1 tape.""" + """Same-code invariants all hold on the original live run's tape.""" evidence = null_control(l1, WORKTREE) invariants = evidence["invariants"] assert invariants["a_prefix_byte_identical"] is True @@ -165,7 +166,7 @@ def test_null_control_placebo_inert(l1): "fa30e29786877c6534013a2f6a2683a6d19e1cb136a66830d35cd9a8defdd582" ) assert evidence["event_counts"] == {"treatment": 127, "placebo": 126, "l1": 126} - # Structurally unable to claim byte-identity to historical L1. + # Structurally unable to claim byte-identity to the historical original run. assert "byte_identical" not in evidence assert "RETIRED" in evidence["standard"] assert evidence["version_drift"]["l1_git_sha"] != evidence["version_drift"][ @@ -184,7 +185,7 @@ def test_null_control_catches_divergence(l1, monkeypatch): post-branch event whose payload no longer matches the placebo fold, which invariant (b) (aligned suffix, non-bookkeeping fields) detects. """ - from verdi.experiments import g5_pairs + from verdi_agents.experiments import counterfactual_pairs as g5_pairs real_run_arm = g5_pairs.run_arm @@ -229,7 +230,7 @@ def test_null_control_symmetric_input_corruption_is_not_divergence(l1): def test_null_control_catches_state_divergence(l1, monkeypatch): """Invariant (c): a fold that changes the final state hash must fail.""" - from verdi.experiments import g5_pairs + from verdi_agents.experiments import counterfactual_pairs as g5_pairs real_run_arm = g5_pairs.run_arm @@ -285,7 +286,7 @@ def test_scaffold_invariance_final_state_hash(arms): def test_arm_ledger_verifies_and_counts(arms): tr, pr = arms - # Placebo arm: byte-identical to L1 (126 events, dismissal = 1 event). + # Placebo arm: byte-identical to the original run (126 events, dismissal = 1 event). # Treatment arm: 127 events (upheld + rights_restored = 2 events at the # branch position, no dismissal). tr.log.verify() @@ -339,14 +340,14 @@ def test_report_byte_stability(l1, arms, context): b1 = json.dumps(r1, indent=2, sort_keys=True) + "\n" b2 = json.dumps(r2, indent=2, sort_keys=True) + "\n" assert b1 == b2 - # No NEW causal wording beyond the three frozen negated occurrences. + # No NEW causal wording beyond the three fixed negated occurrences. assert len(re.findall(r"causal", b1, flags=re.IGNORECASE)) == 3 - # ADR-013 §1 extensions present. + # Extended report sections present. assert "null_control_placebo_inert" in r1 assert r1["matching"]["origin_state_hash"] == context["origin_state_hash"] assert r1["replication"]["n_pairs"] == 10 assert r1["design"]["randomized"] is False - # Addendum A1: numeric cross-pair replication spread (replicated single + # Numeric cross-pair replication spread (replicated single # pair → zero spread, labeled as replication spread). spread = r1["cross_pair_replication_spread"] assert spread["n_pairs"] == 10 @@ -354,7 +355,7 @@ def test_report_byte_stability(l1, arms, context): assert set(spread["values"]) == set(ESTIMANDS) for per_arm in spread["values"].values(): assert per_arm["treatment"] == 0.0 and per_arm["placebo"] == 0.0 - # Addendum A1 framings + corrected limitations (stale entries removed). + # Framings + corrected limitations (stale entries removed). assert "false_punishment_direction" in r1["framings"] assert "deprivation_metric_restoration_blind" in r1["framings"] assert not any("no live LLM interaction" in s for s in r1["limitations"]) diff --git a/tests/experiments/test_integration.py b/tests/experiments/test_integration.py index 3070782..99131e5 100644 --- a/tests/experiments/test_integration.py +++ b/tests/experiments/test_integration.py @@ -1,17 +1,17 @@ -"""P2 WS-2 integration test: pilot -> branch point -> treated vs placebo +"""Integration test: reference run -> branch point -> treated vs placebo branch arms -> matched pair. -Runs the recorded-output pilot (no LLM), forks both arms from the SAME -sealed snapshot at the pilot's governance branch point, and validates: +Runs the recorded-output reference run (no LLM), forks both arms from the SAME +sealed snapshot at its governance branch point, and validates: -* the placebo branch reproduces the pilot's final state hash exactly +* the placebo branch reproduces the reference run's final state hash exactly (the placebo is truly inert); * the rights-restoration treatment's honest ``applied`` flag on this tape - (the pilot appeal is DISMISSED, so no sealed ``rights_restored`` event + (the reference-run appeal is DISMISSED, so no sealed ``rights_restored`` event exists to re-record -> ``applied=False``, tape returned unmodified); -* a :class:`~verdi.experiments.MatchedPair` built from both arms' +* a :class:`~verdi_agents.experiments.MatchedPair` built from both arms' manifests (``branch_origin`` / ``treatment_seed`` carried through via - :func:`verdi.world.branch.manifest_for_branch`) validates because both + :func:`verdi_agents.world.branch.manifest_for_branch`) validates because both arms share ``origin_state_hash`` and the tape hash. """ from __future__ import annotations @@ -19,22 +19,22 @@ import hashlib from pathlib import Path -from verdi.experiments.pairs import MatchedPair, make_matched_pair, tape_hash -from verdi.experiments.treatments import ( +from verdi_agents.experiments.pairs import MatchedPair, make_matched_pair, tape_hash +from verdi_agents.experiments.treatments import ( TreatmentOutcome, placebo_treatment, rights_restoration_treatment, ) -from verdi.pilot.driver import PilotDriver -from verdi.world.branch import BranchSpec, branch_from_snapshot, manifest_for_branch -from verdi.world.reducer import WorldReducer -from verdi.world.replay import replay_in_memory +from verdi_agents.reference_run.driver import PilotDriver +from verdi_agents.world.branch import BranchSpec, branch_from_snapshot, manifest_for_branch +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.replay import replay_in_memory REPO_ROOT = Path(__file__).resolve().parents[2] def _branch_point(): - """Replay the pilot tape up to the first governance step and return + """Replay the reference-run tape up to the first governance step and return (pilot, snapshot, prefix_log, remaining_tape).""" pilot = PilotDriver(REPO_ROOT).run() tape = pilot.full_tape_steps @@ -73,7 +73,7 @@ def test_pilot_placebo_branch_reproduces_final_hash_and_pair_validates(): snapshot, prefix_log, placebo_outcome.tape, spec ) - # (d) the placebo branch must reproduce the pilot final hash exactly. + # (d) the placebo branch must reproduce the reference run's final hash exactly. assert placebo_result.branch_state_hash == pilot.final_state.state_hash # byte-identical full ledger as well (true placebo, not just equal hash) assert placebo_result.log.to_jsonl_bytes() == pilot.log.to_jsonl_bytes() @@ -83,7 +83,7 @@ def test_pilot_placebo_branch_reproduces_final_hash_and_pair_validates(): treatment_spec, remaining, seed=treatment_spec.treatment_seed ) assert isinstance(treatment_outcome, TreatmentOutcome) - # Honest: the pilot tape contains sanction -> appeal_filed -> + # Honest: the reference-run tape contains sanction -> appeal_filed -> # appeal_dismissed and NO rights_restored event, so there is no sealed # restoration outcome to re-record. The treatment must NOT invent one. assert treatment_outcome.applied is False @@ -110,7 +110,7 @@ def test_pilot_placebo_branch_reproduces_final_hash_and_pair_validates(): treatment_manifest = manifest_for_branch(treatment_result.spec, base_manifest) placebo_manifest = manifest_for_branch(placebo_result.spec, base_manifest) - # branch identity is carried into the manifests (B5) + # branch identity is carried into the manifests assert treatment_manifest["branch_origin"] == treatment_spec.branch_id assert treatment_manifest["treatment_seed"] == treatment_spec.treatment_seed assert placebo_manifest["branch_origin"] == spec.branch_id diff --git a/tests/experiments/test_g5_close.py b/tests/experiments/test_pair_audit.py similarity index 93% rename from tests/experiments/test_g5_close.py rename to tests/experiments/test_pair_audit.py index 6a08270..f0978cc 100644 --- a/tests/experiments/test_g5_close.py +++ b/tests/experiments/test_pair_audit.py @@ -1,4 +1,4 @@ -"""Offline tests for verdi.experiments.g5_close (ADR-014 §2/§4/§5/§6). +"""Offline tests for verdi_agents.experiments.pair_audit (audits and report for paired runs). No network, no .env reads. Real-run-root tests skip when the gitignored production run root is absent; a synthetic 2-pair mini-run carries the @@ -18,8 +18,8 @@ requires_real_run_root, ) -from verdi.audit.judge_live import _assert_no_key_material -from verdi.experiments.g5_close import ( +from verdi_agents.audit.judge import _assert_no_key_material +from verdi_agents.experiments.pair_audit import ( ADR5_OVERSTATEMENT_LIMITATION, B1_SCOPE, DEVIATIONS_FROM_CONTRACT, @@ -103,7 +103,7 @@ def _report_inputs(mini_run_root=None): # --------------------------------------------------------------------------- # -# Sealed sample manifest (ADR-014 §2) +# Sealed sample manifest # --------------------------------------------------------------------------- # @@ -148,7 +148,7 @@ def test_sample_manifest_mini_run(tmp_path): # --------------------------------------------------------------------------- # -# Bias / leakage audits (ADR-014 §4): clean mini-run passes, corruptions fail +# Bias / leakage audits (B1..B4): clean mini-run passes, corruptions fail # --------------------------------------------------------------------------- # @@ -160,9 +160,9 @@ def test_audits_pass_on_clean_mini_run(mini_run_root): def test_b1_scope_string_and_bounds_flag(mini_run_root): - # Reviewer F2: B1 must carry the frozen scope string and the + # The self-report audit (B1) must carry its fixed scope string and the # bounds_check_implemented=false flag so the report cannot be misread - # as the full ADR B1. + # as a full self-report check. res = audit_self_report(mini_run_root) assert res["scope"] == B1_SCOPE assert res["bounds_check_implemented"] is False @@ -273,7 +273,7 @@ def test_b3_asymmetric_quarantine_fails_symmetry(mini_run_root): def test_b4_n_suffix_rows_counts_actual_rows(mini_run_root): - # Reviewer MED-1: n_suffix_rows is the ACTUAL count of tape rows with + # n_suffix_rows is the ACTUAL count of tape rows with # tick >= 13 (mini-run: 1 per pair x 2 pairs = 2), not a # distinct-value estimate. res = audit_prompt_invariance(mini_run_root) @@ -282,7 +282,7 @@ def test_b4_n_suffix_rows_counts_actual_rows(mini_run_root): @requires_real_run_root def test_b4_n_suffix_rows_real_run_root(): - # Reviewer MED-1: the real run root has 470 suffix tape rows (9 pairs + # The real run root has 470 suffix tape rows (9 pairs # x 47 + pair_0's 27 suffix rows). The former # sum(distinct values)//3 estimate reported 25. res = audit_prompt_invariance(REAL_G5_RUN_ROOT) @@ -290,7 +290,7 @@ def test_b4_n_suffix_rows_real_run_root(): def test_b2_method_text_discloses_json_escaping(mini_run_root): - # Reviewer F5: B2's method string must disclose that JSON-escaped + # The leakage audit's (B2) method string must disclose that JSON-escaped # forms defeat verbatim containment. res = audit_leakage(mini_run_root) assert "JSON-escaped forms" in res["method"] @@ -313,7 +313,7 @@ def test_b4_prompt_drift_fails_invariance(mini_run_root): # --------------------------------------------------------------------------- # -# Identification diagnostics (ADR-014 §5) +# Identification diagnostics # --------------------------------------------------------------------------- # @@ -349,7 +349,7 @@ def test_permutation_null_real_run_root_symmetry(): assert block["contains_zero_mean"] or abs(block["min"] + block["max"]) <= 1e-12 fp = res["per_estimand"]["false_punishment_rate"] assert fp["magnitudes_equal"] is True - # Reviewer F4 (corrected reasoning): the fixed seed 0 makes the 200 + # The fixed seed 0 makes the 200 # draws DETERMINISTIC. A permutation mean is exactly 0.0 iff exactly # 5 of the 10 signs are negative; 57 of the 200 recorded draws have # exactly 5 negative signs, so 0.0 appears in exactly those 57. The @@ -389,7 +389,7 @@ def test_leave_one_pair_out_and_permutation_minimal(tmp_path): # --------------------------------------------------------------------------- # -# Report (ADR-014 §6) +# Report # --------------------------------------------------------------------------- # @@ -417,7 +417,7 @@ def test_full_report_real_run_root(tmp_path): } assert report["identification"]["not_claimed"] == list(NOT_CLAIMED) assert report["limitations"] == list(FROZEN_LIMITATIONS) - # Reviewer F3: the ADR-014 §5 overstatement is NOT inherited. + # The report must not repeat the design's overstated identification claim. assert ADR5_OVERSTATEMENT_LIMITATION in report["limitations"] framing = report["judge"]["framing"] assert framing["judge_shown_engine_outcome"] is True @@ -435,7 +435,7 @@ def test_full_report_real_run_root(tmp_path): def test_deviations_from_contract_frozen_verbatim(tmp_path): - # Reviewer MED-2 / F1: the top-level deviations block is a frozen list, + # The top-level deviations block is a fixed list, # carried verbatim into every report. inputs = _report_inputs(mini_run_root=make_mini_run_root(tmp_path / "m")) report = build_g5_close_report( @@ -453,7 +453,7 @@ def test_deviations_from_contract_frozen_verbatim(tmp_path): def test_adr5_overstatement_limitation_verbatim(): - # Reviewer F3: the exact frozen wording, tested verbatim. + # The exact fixed wording, tested verbatim. assert ADR5_OVERSTATEMENT_LIMITATION == ( "ADR-014 §5's 'evidence the contrast is tied to the manipulation' " "phrasing is deliberately not inherited by the report; the " @@ -550,12 +550,12 @@ def test_report_rejects_unfrozen_not_claimed_block(mini_run_root): def test_no_key_material_in_tape_and_report(mini_run_root): fake_key = "sk-fakekey99aabbccddeeff" - # The key-material scanner (judge_live behavior) must raise on the tape + # The key-material scanner (same behavior as in verdi_agents.audit.judge) must raise on the tape # text that would carry a fake key ... with pytest.raises(AssertionError): _assert_no_key_material(f'"raw": "verdict {fake_key} end"', "fake tape") # ... but the clean artifacts of a mock judge run must NOT raise. - from verdi.experiments.g5_judge_live import MockJudgeClient, run_g5_judge + from verdi_agents.experiments.pair_judge import MockJudgeClient, run_g5_judge run = run_g5_judge( mini_run_root, @@ -570,7 +570,7 @@ def test_no_key_material_in_tape_and_report(mini_run_root): _assert_no_key_material(tape_text, "judge_tape.jsonl") _assert_no_key_material(meta_text, "g5judge_run_meta.json") # A report containing a fake key must fail the write guard. - from verdi.experiments.g5_close import write_g5_close_report + from verdi_agents.experiments.pair_audit import write_g5_close_report inputs = _report_inputs(mini_run_root=mini_run_root) report = build_g5_close_report( @@ -642,14 +642,14 @@ def test_build_judge_section_agreement_and_enumeration(): # --------------------------------------------------------------------------- # -# build_judge_section: reviewer F1 (code_judge), F7 (row_provenance), -# F8 (engine-outcome n<=1 flag) +# build_judge_section: code_judge, row_provenance, and the +# engine-outcome n<=1 flag # --------------------------------------------------------------------------- # def test_judge_section_code_judge_recorded_unavailable(): - # Reviewer F1 / MED-2: null-free, explicit record that the code-judge - # stance comparison is not possible on the G5 run root. + # Null-free, explicit record that the code-judge stance comparison + # is not possible on the paired-arms run root. manifest = {"run_root": "x", "n_rows": 1, "manifest_sha256": "0" * 64} section = build_judge_section([GOOD_VERDICT_ROW], manifest) assert section["code_judge"] == { @@ -665,7 +665,7 @@ def test_judge_section_code_judge_recorded_unavailable(): def test_judge_section_row_provenance_computed_from_sample(tmp_path): - # Reviewer F7: counts are computed from the sample, never hardcoded. + # Counts are computed from the sample, never hardcoded. root = make_mini_run_root(tmp_path / "m") manifest = build_sample_manifest(root) # Mini sample: pair_0 all 3 rows (ticks 1, 2, 13) + pair_1 tick 13 @@ -681,8 +681,9 @@ def test_judge_section_row_provenance_computed_from_sample(tmp_path): @requires_real_run_root def test_judge_section_row_provenance_real_run_root(): - # Reviewer F7: on the real sealed sample, 470 fresh suffix rows and 34 - # L1-replayed prefix rows (counts computed, not hardcoded). + # On the real sealed sample: 470 freshly generated suffix rows and 34 + # prefix rows replayed from the original live run (counts computed, + # not hardcoded). manifest = build_sample_manifest(REAL_G5_RUN_ROOT) section = build_judge_section([GOOD_VERDICT_ROW], manifest) prov = section["row_provenance"] @@ -691,8 +692,8 @@ def test_judge_section_row_provenance_real_run_root(): def test_judge_section_engine_outcome_n1_flag(tmp_path): - # Reviewer F8: data-driven flag on any engine-outcome class with - # n <= 1 (G4 convention); no class/event is hardcoded. + # Data-driven flag on any engine-outcome class with n <= 1 (too few + # rows to be statistically meaningful); no class/event is hardcoded. root = make_mini_run_root(tmp_path / "m") manifest = build_sample_manifest(root) good = dict(GOOD_VERDICT_ROW) @@ -731,9 +732,9 @@ def test_judge_section_engine_outcome_n1_flag(tmp_path): @requires_real_run_root def test_judge_section_engine_outcome_breakdown_real_run_root(): - # Reviewer F8: the real tape maps 503 accepted / 1 rejected + # The real tape maps 503 accepted / 1 rejected # (pair_0 event 42); the n=1 rejected class carries the flag. - from verdi.experiments.g5_judge_live import MockJudgeClient, run_g5_judge + from verdi_agents.experiments.pair_judge import MockJudgeClient, run_g5_judge root = REAL_G5_RUN_ROOT out_dir = root / "g5judge_test_section" diff --git a/tests/experiments/test_g5_judge_live.py b/tests/experiments/test_pair_judge.py similarity index 91% rename from tests/experiments/test_g5_judge_live.py rename to tests/experiments/test_pair_judge.py index 73cd25b..5099260 100644 --- a/tests/experiments/test_g5_judge_live.py +++ b/tests/experiments/test_pair_judge.py @@ -1,7 +1,7 @@ -"""Offline tests for verdi.experiments.g5_judge_live (ADR-014 §3/§6). +"""Offline tests for verdi_agents.experiments.pair_judge, the LLM judge for paired runs. -All network is mocked via :class:`MockJudgeClient` (ADR-012 -``_RecordingJudgeClient`` pattern). No network, no .env reads. +All network is mocked via :class:`MockJudgeClient` (same pattern as +``_RecordingJudgeClient`` in verdi_agents.audit.judge). No network, no .env reads. """ from __future__ import annotations @@ -12,8 +12,8 @@ import pytest from tests.experiments.conftest import make_mini_run_root -from verdi.experiments.g5_close import build_sample_manifest -from verdi.experiments.g5_judge_live import ( +from verdi_agents.experiments.pair_audit import build_sample_manifest +from verdi_agents.experiments.pair_judge import ( JUDGE_SYSTEM_PROMPT, MockJudgeClient, engine_outcome_for_row, @@ -127,7 +127,7 @@ def test_run_g5_judge_mock_tape_and_meta(mini_run_root): } first = sample_rows[0] assert first["prompt"][0]["content"] == JUDGE_SYSTEM_PROMPT - # judge is SHOWN the engine outcome (ADR-014 §3 disclosure) + # the judge is SHOWN the engine outcome (this is disclosed in the report) assert "engine_outcome: decision=accepted" in first["prompt"][1]["content"] assert "MODEL RESPONSE (sealed, verbatim)" in first["prompt"][1]["content"] assert first["model"] == "mock-judge" @@ -182,7 +182,7 @@ def test_run_g5_judge_cap_abort(mini_run_root): def test_run_g5_judge_cap_abort_records_meta(mini_run_root): - # Reviewer MED-4 / F6: on cap breach the run meta is written best-effort + # On cap breach the run meta is written best-effort # with status "cap_aborted" BEFORE the abort raises. out_dir = mini_run_root.parent / "judge_out_cap" with pytest.raises(Exception, match="hard call cap"): @@ -202,7 +202,7 @@ def test_run_g5_judge_cap_abort_records_meta(mini_run_root): def test_mock_client_max_calls_enforced(): - # Reviewer MED-3: MockJudgeClient.max_calls is the tripwire that would + # MockJudgeClient.max_calls is the tripwire that would # have caught a client-internal double-call. The 1st call succeeds; # the 2nd raises. import asyncio @@ -224,10 +224,10 @@ async def _two_calls(): def test_endpoint_config_zero_retries(monkeypatch): - # Reviewer HIGH-1: the effective endpoint config MUST carry + # The effective endpoint config MUST carry # max_retries=0 so client-internal retries cannot blow the hard cap. - from verdi.config import ModelEndpointConfig - from verdi.experiments.g5_judge_live import _resolve_endpoint_config + from verdi_agents.config import ModelEndpointConfig + from verdi_agents.experiments.pair_judge import _resolve_endpoint_config monkeypatch.setenv("VERDI_DOTENV_PATH", "/nonexistent/.env") monkeypatch.setenv("VERDI_VLLM_BASE_URL", "http://localhost:9999/v1") @@ -244,7 +244,7 @@ def test_run_g5_judge_sample_count_assertion(mini_run_root): # The mini run has 4 sample rows, not the frozen production 504: the # default assert fires with a clear message (run_g5_judge is built for # the sealed production sample). - from verdi.experiments.g5_judge_live import G5JudgeError + from verdi_agents.experiments.pair_judge import G5JudgeError with pytest.raises(G5JudgeError, match="expected 504"): run_g5_judge( @@ -267,7 +267,7 @@ def test_load_dotenv_handles_export_prefix(tmp_path, monkeypatch): ) monkeypatch.delenv("VERDI_TEST_BASE_URL", raising=False) monkeypatch.setenv("VERDI_DOTENV_PATH", str(env_file)) - from verdi.experiments.g5_judge_live import _load_dotenv_into_environ + from verdi_agents.experiments.pair_judge import _load_dotenv_into_environ _load_dotenv_into_environ() assert os.environ["VERDI_TEST_BASE_URL"] == "https://example/api/v1" diff --git a/tests/experiments/test_paired_run.py b/tests/experiments/test_paired_run.py index 3f4e20f..0e574bc 100644 --- a/tests/experiments/test_paired_run.py +++ b/tests/experiments/test_paired_run.py @@ -1,4 +1,4 @@ -"""Offline tests for the P0-C paired fixed-vs-revisable runner. +"""Offline tests for the paired runner that compares fixed and revisable rules. Uses a scripted fake caller (pattern from tests/society/test_campaign.py): producers produce food, depositors top up the commons, assembly voters vote @@ -15,8 +15,8 @@ import pytest import yaml -from verdi.agents.client import ModelResult -from verdi.experiments.paired_run import ( +from verdi_agents.agents.client import ModelResult +from verdi_agents.experiments.paired_run import ( build_arm_scenarios, load_paired_config, run_pair, @@ -366,7 +366,7 @@ def test_build_arm_scenarios_asserts_identity(): def test_cli_main_validates_before_endpoint(tmp_path, monkeypatch): """main() must reject a bad config BEFORE any endpoint resolution.""" - from verdi.experiments import paired_run + from verdi_agents.experiments import paired_run base_file = _write_base(tmp_path) bad_cfg = tmp_path / "bad.yaml" @@ -396,7 +396,7 @@ def test_example_config_is_valid_and_self_consistent(): assert config["seeds"] == [42] assert (repo_root / config["base_scenario"]).exists() # Identity holds for the real smoke scenario too. - from verdi.analysis.scenario import load_scenario + from verdi_agents.analysis.scenario import load_scenario base = load_scenario(repo_root / config["base_scenario"]) built = build_arm_scenarios(base, config["arms"]) minus_a = {k: v for k, v in built["fixed"].items() if k != "institutional_condition"} diff --git a/tests/experiments/test_pairs.py b/tests/experiments/test_pairs.py index 332995b..128bccf 100644 --- a/tests/experiments/test_pairs.py +++ b/tests/experiments/test_pairs.py @@ -8,13 +8,13 @@ import pytest from tests.world.helpers import initial_world -from verdi.experiments.pairs import MatchedPair, make_matched_pair, tape_hash -from verdi.schemas import ProposedAction, ValidationResult -from verdi.world.branch import BranchSpec, branch_from_snapshot -from verdi.world.canonical import canonical_json -from verdi.world.log import EventLog -from verdi.world.reducer import WorldReducer -from verdi.world.replay import TapeStep, replay_in_memory +from verdi_agents.experiments.pairs import MatchedPair, make_matched_pair, tape_hash +from verdi_agents.schemas import ProposedAction, ValidationResult +from verdi_agents.world.branch import BranchSpec, branch_from_snapshot +from verdi_agents.world.canonical import canonical_json +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.replay import TapeStep, replay_in_memory def _prefix_log() -> EventLog: @@ -109,7 +109,7 @@ def test_end_to_end_pair_from_branch_results(): origin_state_hash=origin_hash, origin_event_count=len(log), origin_tick=snapshot.tick, branch_id="placebo", treatment_seed=0, )) - from verdi.world.branch import manifest_for_branch + from verdi_agents.world.branch import manifest_for_branch treatment = manifest_for_branch(res_t.spec, base) placebo = manifest_for_branch(res_p.spec, base) diff --git a/tests/experiments/test_regimes.py b/tests/experiments/test_regimes.py index c0a1394..f8e6ca8 100644 --- a/tests/experiments/test_regimes.py +++ b/tests/experiments/test_regimes.py @@ -6,15 +6,15 @@ import pytest from pydantic import ValidationError -from verdi.experiments.regimes import ( +from verdi_agents.experiments.regimes import ( REGIME_LABELS, RegimeConfig, load_regime, manifest_metadata, ) -REGIMES_DIR = Path(__file__).resolve().parents[2] / "src" / "verdi" / "experiments" / "regimes" -REGIME_FILES = ("g0_baseline.json", "g1_single_adversary.json", "g3_detection_on.json") +REGIMES_DIR = Path(__file__).resolve().parents[2] / "src" / "verdi_agents" / "experiments" / "regimes" +REGIME_FILES = ("baseline.json", "single_adversary.json", "detection_on.json") def test_fixture_regimes_validate(): @@ -29,9 +29,9 @@ def test_fixture_regimes_validate(): def test_fixture_regimes_cover_required_casemix(): - g0 = load_regime(REGIMES_DIR / "g0_baseline.json") - g1 = load_regime(REGIMES_DIR / "g1_single_adversary.json") - g3 = load_regime(REGIMES_DIR / "g3_detection_on.json") + g0 = load_regime(REGIMES_DIR / "baseline.json") + g1 = load_regime(REGIMES_DIR / "single_adversary.json") + g3 = load_regime(REGIMES_DIR / "detection_on.json") # baseline: no adversary, everything off assert g0.adversary_count == 0 assert (g0.reflection, g0.memory, g0.detection) == (False, False, False) @@ -76,7 +76,7 @@ def test_regime_is_frozen_and_no_extra_fields(): def test_manifest_metadata_round_trips_without_schema_change(): - config = load_regime(REGIMES_DIR / "g3_detection_on.json") + config = load_regime(REGIMES_DIR / "detection_on.json") metadata = manifest_metadata(config) assert metadata["regime_label"] == config.regime assert RegimeConfig.model_validate(metadata["regime"]) == config diff --git a/tests/experiments/test_sweep.py b/tests/experiments/test_sweep.py index 8301752..b9cd6e4 100644 --- a/tests/experiments/test_sweep.py +++ b/tests/experiments/test_sweep.py @@ -14,9 +14,9 @@ import pytest import yaml -from verdi.analysis.scenario import build_t0_state -from verdi.experiments import sweep as sw -from verdi.society.campaign import SocietyCampaign +from verdi_agents.analysis.scenario import build_t0_state +from verdi_agents.experiments import sweep as sw +from verdi_agents.society.campaign import SocietyCampaign REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT / "tests")) @@ -213,7 +213,7 @@ def test_shipped_example_sweeps_validate(name, tmp_path, capsys): def test_verdi_cli_routes_sweep(capsys, tmp_path): - from verdi.cli import main + from verdi_agents.cli import main assert main(["sweep", "--scenario", STARTER, "--output", str(tmp_path), "--dry-run"]) == 0 assert "Sweep:" in capsys.readouterr().out diff --git a/tests/experiments/test_treatments.py b/tests/experiments/test_treatments.py index bf8996d..6542382 100644 --- a/tests/experiments/test_treatments.py +++ b/tests/experiments/test_treatments.py @@ -6,17 +6,17 @@ import pytest from tests.world.helpers import initial_world -from verdi.experiments.treatments import ( +from verdi_agents.experiments.treatments import ( TreatmentOutcome, placebo_treatment, rights_restoration_treatment, ) -from verdi.schemas import ProposedAction, ValidationResult -from verdi.world.branch import BranchSpec, branch_from_snapshot -from verdi.world.canonical import canonical_sha256 -from verdi.world.log import EventLog -from verdi.world.reducer import WorldReducer -from verdi.world.replay import TapeStep +from verdi_agents.schemas import ProposedAction, ValidationResult +from verdi_agents.world.branch import BranchSpec, branch_from_snapshot +from verdi_agents.world.canonical import canonical_sha256 +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.replay import TapeStep class _Spec: @@ -126,7 +126,7 @@ def test_placebo_deterministic_across_seeds(): # rights restoration # --------------------------------------------------------------------------- # def test_rights_restoration_reports_unapplied_when_no_restoration_in_tape(): - tape = _base_tape() # the pilot's shape: sanction -> appeal -> dismissed + tape = _base_tape() # the reference run's shape: sanction -> appeal -> dismissed fingerprint_before = _tape_fingerprint(tape) outcome = rights_restoration_treatment(_Spec(), tape, seed=0) assert isinstance(outcome, TreatmentOutcome) diff --git a/experiments/scenarios/pilot_v1.json b/tests/fixtures/reference_run/scenarios/pilot_v1.json similarity index 100% rename from experiments/scenarios/pilot_v1.json rename to tests/fixtures/reference_run/scenarios/pilot_v1.json diff --git a/experiments/task_tape/pilot_v1.json b/tests/fixtures/reference_run/task_tape/pilot_v1.json similarity index 100% rename from experiments/task_tape/pilot_v1.json rename to tests/fixtures/reference_run/task_tape/pilot_v1.json diff --git a/tests/fixtures/scenarios/minimal.yaml b/tests/fixtures/scenarios/minimal.yaml index c294a16..8c07cb6 100644 --- a/tests/fixtures/scenarios/minimal.yaml +++ b/tests/fixtures/scenarios/minimal.yaml @@ -5,7 +5,7 @@ # few numbers, run the command below, then open the results folder. # # source .venv/bin/activate -# python -m verdi.society.live \ +# python -m verdi_agents.society.live \ # --scenario tests/fixtures/scenarios/minimal.yaml \ # --run-dir runs/my-run \ # --concurrency 4 diff --git a/tests/governance/conftest.py b/tests/governance/conftest.py index cd4ddeb..01352cf 100644 --- a/tests/governance/conftest.py +++ b/tests/governance/conftest.py @@ -3,8 +3,8 @@ from typing import Any -from verdi.governance import GovernanceEngine -from verdi.schemas import ( +from verdi_agents.governance import GovernanceEngine +from verdi_agents.schemas import ( Appeal, ConstitutionVersion, EvidenceReference, diff --git a/tests/governance/test_amendment_end_to_end.py b/tests/governance/test_amendment_end_to_end.py index a538119..21ac6e9 100644 --- a/tests/governance/test_amendment_end_to_end.py +++ b/tests/governance/test_amendment_end_to_end.py @@ -22,10 +22,10 @@ import json from typing import Any -from verdi.agents.client import ModelResult -from verdi.governance import ConstitutionStore, UniversalLaw -from verdi.schemas import ConstitutionVersion, LawVersion -from verdi.society.campaign import SocietyCampaign +from verdi_agents.agents.client import ModelResult +from verdi_agents.governance import ConstitutionStore, UniversalLaw +from verdi_agents.schemas import ConstitutionVersion, LawVersion +from verdi_agents.society.campaign import SocietyCampaign ASSEMBLY_TICK = 15 AGENTS = ("alice", "bob", "chen", "dave") @@ -179,8 +179,8 @@ def test_enacted_amendment_flips_subsequent_validation_decision(tmp_path): # 4) the rule is enforced EXACTLY at the cap: a 5-unit transfer under the # enacted rules is still accepted (the campaign passes exactly these # active-version rules to validate_action each tick). - from verdi.schemas import ProposedAction - from verdi.world.validate import validate_action + from verdi_agents.schemas import ProposedAction + from verdi_agents.world.validate import validate_action # constitution_texts: one full model_dump per registered version entry = next(t for t in res.constitution_texts diff --git a/tests/governance/test_appeals.py b/tests/governance/test_appeals.py index 1ee1b3a..38cfd84 100644 --- a/tests/governance/test_appeals.py +++ b/tests/governance/test_appeals.py @@ -3,8 +3,8 @@ import pytest -from verdi.governance import AppealEngine, SanctionEngine -from verdi.governance.appeals import AppealError +from verdi_agents.governance import AppealEngine, SanctionEngine +from verdi_agents.governance.appeals import AppealError from .conftest import make_appeal, make_sanction diff --git a/tests/governance/test_assembled.py b/tests/governance/test_assembled.py index 39c6156..2ffb450 100644 --- a/tests/governance/test_assembled.py +++ b/tests/governance/test_assembled.py @@ -1,4 +1,4 @@ -"""t10 real-assembly driver: model-voted proposals and votes (Society V2 §1.2). +"""Tests for the real assembly: model-written proposals and model votes. Uses a FAKE ModelCaller (canned JSON, no network) — the Assembly's job is to route model output through the deterministic engine, never to decide outcomes @@ -11,8 +11,8 @@ import json import re -from verdi.governance.assembled import Assembly -from verdi.schemas import AgentState, ValueProfile, WorldState +from verdi_agents.governance.assembly import Assembly +from verdi_agents.schemas import AgentState, ValueProfile, WorldState from .conftest import ELIGIBLE, make_engine @@ -239,8 +239,8 @@ def test_parallel_batch_runner_matches_sequential_outcomes(): proposals, votes, quarantine records, abstentions and outcome as the default sequential runner for identical model outputs — only call wall-clock/telemetry order may differ.""" - from verdi.governance.assembled import _run_sequential - from verdi.society.campaign import _run_bounded + from verdi_agents.governance.assembly import _run_sequential + from verdi_agents.society.campaign import _run_bounded def handler(phase, voter, proposal): if phase == "delib": diff --git a/tests/governance/test_audit_regressions.py b/tests/governance/test_audit_regressions.py index 07e06a2..3c2122f 100644 --- a/tests/governance/test_audit_regressions.py +++ b/tests/governance/test_audit_regressions.py @@ -1,11 +1,11 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import pytest -from verdi.governance import ConstitutionStore, SanctionEngine, UniversalLaw -from verdi.governance.appeals import AppealEngine -from verdi.governance.constitution import ProposalNotInVoteError +from verdi_agents.governance import ConstitutionStore, SanctionEngine, UniversalLaw +from verdi_agents.governance.appeals import AppealEngine +from verdi_agents.governance.constitution import ProposalNotInVoteError from .conftest import CONSTITUTION, LAW, make_sanction, make_vote diff --git a/tests/governance/test_constitution.py b/tests/governance/test_constitution.py index e5a71f0..ec1c3a7 100644 --- a/tests/governance/test_constitution.py +++ b/tests/governance/test_constitution.py @@ -3,8 +3,8 @@ import pytest -from verdi.governance import ConstitutionStore, UniversalLaw -from verdi.governance.constitution import UnknownProposalError +from verdi_agents.governance import ConstitutionStore, UniversalLaw +from verdi_agents.governance.constitution import UnknownProposalError from .conftest import CONSTITUTION, LAW, make_vote @@ -122,7 +122,7 @@ def test_effective_at_current_tick_allowed(): def test_double_vote_rejected_deterministically(): - from verdi.governance.constitution import DuplicateVoteError + from verdi_agents.governance.constitution import DuplicateVoteError store = _store() store.propose("p1", "a1", 10, "tweak", {"tax_rate": 0.2}) diff --git a/tests/governance/test_determinism.py b/tests/governance/test_determinism.py index 55e3a58..caa65b1 100644 --- a/tests/governance/test_determinism.py +++ b/tests/governance/test_determinism.py @@ -9,9 +9,9 @@ def _imported_modules() -> dict[str, list[str]]: """Map each governance module to the module names it imports (via AST).""" - package = importlib.import_module("verdi.governance") + package = importlib.import_module("verdi_agents.governance") result: dict[str, list[str]] = {} - for mod in pkgutil.walk_packages(package.__path__, prefix="verdi.governance."): + for mod in pkgutil.walk_packages(package.__path__, prefix="verdi_agents.governance."): module = importlib.import_module(mod.name) source_file = Path(module.__file__) tree = ast.parse(source_file.read_text(encoding="utf-8")) @@ -39,8 +39,8 @@ def test_no_llm_or_nondeterministic_imports(): def test_engine_source_has_no_llm_markers(): """Source must contain no model/LLM invocation markers.""" markers = ("chat.completions", "vllm", "model.infer", "openai") - package = importlib.import_module("verdi.governance") - for mod in pkgutil.walk_packages(package.__path__, prefix="verdi.governance."): + package = importlib.import_module("verdi_agents.governance") + for mod in pkgutil.walk_packages(package.__path__, prefix="verdi_agents.governance."): module = importlib.import_module(mod.name) text = Path(module.__file__).read_text(encoding="utf-8").lower() hits = [m for m in markers if m in text] diff --git a/tests/governance/test_engine.py b/tests/governance/test_engine.py index a509b07..3c2bfa6 100644 --- a/tests/governance/test_engine.py +++ b/tests/governance/test_engine.py @@ -1,7 +1,7 @@ -"""GovernanceEngine facade: event emission, lifecycle, determinism (G3).""" +"""GovernanceEngine facade: event emission, lifecycle, determinism.""" from __future__ import annotations -from verdi.governance.engine import GOVERNANCE_EVENT_KINDS +from verdi_agents.governance.engine import GOVERNANCE_EVENT_KINDS from .conftest import ( ELIGIBLE, @@ -104,7 +104,7 @@ def test_sanction_imposed_event_and_evidence_recorded(): def test_sanction_without_evidence_emits_rejected_event(): engine = make_engine() - from verdi.schemas import Sanction + from verdi_agents.schemas import Sanction # Bypass schema validation so the engine-level refusal is exercised. bare = Sanction.model_construct( @@ -172,7 +172,7 @@ def test_appeal_dismissed_emits_event_and_keeps_sanction(): def test_deterministic_replay_same_event_tape(): - """G3: same recorded input tape -> byte-identical governance event log.""" + """Same recorded input tape -> byte-identical governance event log.""" def run() -> list[dict]: engine = make_engine() diff --git a/tests/governance/test_institutional_conditions.py b/tests/governance/test_institutional_conditions.py index 9164486..6da401e 100644 --- a/tests/governance/test_institutional_conditions.py +++ b/tests/governance/test_institutional_conditions.py @@ -1,7 +1,7 @@ """Fixed vs revisable procedures keep the same meeting and voting machinery.""" from __future__ import annotations -from verdi.governance import ConstitutionStore, UniversalLaw +from verdi_agents.governance import ConstitutionStore, UniversalLaw from .conftest import CONSTITUTION, LAW, make_vote diff --git a/tests/governance/test_law.py b/tests/governance/test_law.py index 335d3a3..234d498 100644 --- a/tests/governance/test_law.py +++ b/tests/governance/test_law.py @@ -1,8 +1,8 @@ """Law invariant enforcement tests (deterministic string matching).""" from __future__ import annotations -from verdi.governance import UniversalLaw -from verdi.governance.law import UniversalLaw as LawDirect +from verdi_agents.governance import UniversalLaw +from verdi_agents.governance.law import UniversalLaw as LawDirect from .conftest import LAW diff --git a/tests/governance/test_sanctions.py b/tests/governance/test_sanctions.py index 0436363..b756137 100644 --- a/tests/governance/test_sanctions.py +++ b/tests/governance/test_sanctions.py @@ -3,8 +3,8 @@ import pytest -from verdi.governance import SanctionEngine -from verdi.governance.sanctions import SanctionError +from verdi_agents.governance import SanctionEngine +from verdi_agents.governance.sanctions import SanctionError from .conftest import make_evidence, make_sanction @@ -44,7 +44,7 @@ def test_sanction_without_evidence_refused(): engine = SanctionEngine() from pydantic import ValidationError - from verdi.schemas import Sanction + from verdi_agents.schemas import Sanction # Schema itself refuses empty evidence (frozen contract). with pytest.raises(ValidationError): @@ -68,7 +68,7 @@ def test_sanction_evidence_without_event_ids_refused(): engine = SanctionEngine() # A reference with no cited event ids is evidence in name only -> refused. empty_ev = make_evidence("ev-0", []) - from verdi.schemas import Sanction + from verdi_agents.schemas import Sanction bare = Sanction( sanction_id="s-1", subject_id="a2", evidence=[empty_ev], diff --git a/tests/measurement/test_longitudinal.py b/tests/measurement/test_longitudinal.py index c7a0a82..cc10d4d 100644 --- a/tests/measurement/test_longitudinal.py +++ b/tests/measurement/test_longitudinal.py @@ -1,4 +1,4 @@ -"""Tests for longitudinal live probes (Lane E, Society V2). +"""Tests for longitudinal live probes asked repeatedly during a society run. Covers: row count/channels, per-channel trajectories with no pooled key, yes/no/JSON/garbage parsing, per-channel grounding (persona in every channel; @@ -11,14 +11,14 @@ from dataclasses import dataclass, field from typing import Any -from verdi.measurement.longitudinal import ( +from verdi_agents.measurement.longitudinal import ( ProbePoint, ProbeRecorder, channel_trajectories, ) -from verdi.measurement.probes import CHANNELS, ProbeBank -from verdi.measurement.scoring import score_probe -from verdi.schemas import AgentState, ValueProfile, WorldState +from verdi_agents.measurement.probes import CHANNELS, ProbeBank +from verdi_agents.measurement.scoring import score_probe +from verdi_agents.schemas import AgentState, ValueProfile, WorldState @dataclass(frozen=True) diff --git a/tests/measurement/test_probes.py b/tests/measurement/test_probes.py index fcee401..b9f8154 100644 --- a/tests/measurement/test_probes.py +++ b/tests/measurement/test_probes.py @@ -1,11 +1,11 @@ -"""G0/G1 tests for the hidden probe bank (P1-D, G3 isolation defense).""" +"""Tests for the hidden probe bank and its isolation from the agents.""" from __future__ import annotations import pytest from pydantic import ValidationError -from verdi.measurement import ProbeBank -from verdi.schemas import ProbeItem +from verdi_agents.measurement import ProbeBank +from verdi_agents.schemas import ProbeItem def _bank() -> ProbeBank: diff --git a/tests/measurement/test_scoring.py b/tests/measurement/test_scoring.py index b1c625f..28e6480 100644 --- a/tests/measurement/test_scoring.py +++ b/tests/measurement/test_scoring.py @@ -1,11 +1,11 @@ -"""G0/G1 tests for deterministic option-based scoring (P1-D).""" +"""Tests for deterministic option-based scoring.""" from __future__ import annotations import pytest -from verdi.measurement import JudgeProtocol, RubricScorer, score_probe -from verdi.measurement.probes import ProbeBank -from verdi.schemas import ProbeItem +from verdi_agents.measurement import JudgeProtocol, RubricScorer, score_probe +from verdi_agents.measurement.probes import ProbeBank +from verdi_agents.schemas import ProbeItem def _item() -> ProbeItem: diff --git a/tests/measurement/test_value_drift.py b/tests/measurement/test_value_drift.py index 76006bc..3c58c6a 100644 --- a/tests/measurement/test_value_drift.py +++ b/tests/measurement/test_value_drift.py @@ -1,11 +1,11 @@ -"""G0/G1 tests for per-channel longitudinal drift (P1-D, construct validity).""" +"""Tests for per-channel value drift over time (construct validity).""" from __future__ import annotations import pytest -from verdi.measurement import AxisDrift, bootstrap_ci, compute_channel_drifts -from verdi.measurement.value_drift import compute_axis_drift -from verdi.schemas import ProbeItem, ProbeResult +from verdi_agents.measurement import AxisDrift, bootstrap_ci, compute_channel_drifts +from verdi_agents.measurement.value_drift import compute_axis_drift +from verdi_agents.schemas import ProbeItem, ProbeResult def _item(channel: str, axes: list[str]) -> ProbeItem: diff --git a/tests/pilot/__init__.py b/tests/reference_run/__init__.py similarity index 100% rename from tests/pilot/__init__.py rename to tests/reference_run/__init__.py diff --git a/tests/pilot/test_leak_guard.py b/tests/reference_run/test_leak_guard.py similarity index 83% rename from tests/pilot/test_leak_guard.py rename to tests/reference_run/test_leak_guard.py index a638e01..5048b37 100644 --- a/tests/pilot/test_leak_guard.py +++ b/tests/reference_run/test_leak_guard.py @@ -1,7 +1,7 @@ -"""Regression tests for the G3 probe-leak guard. +"""Regression tests for the probe-leak guard. -The reviewer finding: ``ProbeBank.leak_check`` existed but was never wired -into the pilot run. The guard must (a) pass on the clean pilot run, (b) +``ProbeBank.leak_check`` once existed but was never wired into the +reference run. The guard must (a) pass on the clean reference run, (b) raise ``ProbeLeakError`` when a hidden dilemma text appears in any agent-visible surface, and (c) treat the containment check as verbatim only (paraphrase is documented out of scope). @@ -12,8 +12,8 @@ import pytest -from verdi.measurement.probes import ProbeBank -from verdi.pilot.driver import ( +from verdi_agents.measurement.probes import ProbeBank +from verdi_agents.reference_run.driver import ( BANK_VERSION, INSTRUMENT_VERSION, PilotDriver, @@ -41,7 +41,7 @@ def test_verbatim_leak_raises(bank) -> None: def test_pilot_run_passes_leak_guard() -> None: - """The full pilot run completes, which means the per-tick guard ran over + """The full reference run completes, which means the per-tick guard ran over every agent-visible surface (persona summaries, public signals, public observations, rendered prompts) and found no dilemma text.""" PilotDriver(REPO_ROOT).run() # raises ProbeLeakError on any leak diff --git a/tests/pilot/test_live_scaffold.py b/tests/reference_run/test_live_scaffold.py similarity index 91% rename from tests/pilot/test_live_scaffold.py rename to tests/reference_run/test_live_scaffold.py index 7fcd20c..9a7eeee 100644 --- a/tests/pilot/test_live_scaffold.py +++ b/tests/reference_run/test_live_scaffold.py @@ -1,4 +1,4 @@ -"""L1 live-capture scaffold tests (ADR-010) — NO NETWORK. +"""Tests for live-model capture and tape replay in the reference run (no network). A FakeLiveCaller implements the ModelCaller protocol and stands in for the live Qwen endpoint. These tests pin the live path of PilotDriver: @@ -11,7 +11,7 @@ * live_run_meta.json never contains key material. The REAL live run is a script-style invocation outside pytest; CI must -never make network calls (see src/verdi/pilot/live.py docstring). +never make network calls (see src/verdi_agents/reference_run/live.py docstring). """ from __future__ import annotations @@ -20,24 +20,24 @@ import pytest -from verdi.agents.client import ModelResult -from verdi.pilot.driver import ( +from verdi_agents.agents.client import ModelResult +from verdi_agents.reference_run.driver import ( PilotDriver, _precompute_dispatch, model_output_for, ) -from verdi.scheduler.seeded import SeededScheduler +from verdi_agents.scheduler.seeded import SeededScheduler MAIN_CHECKOUT = Path(__file__).resolve().parents[2] -# ADR-010 replay provenance: the ONE real L1 live run and its captured -# artifacts. Hardcoded absolute paths, skip-if-missing (they are run +# The one recorded live-model run and its captured artifacts, used as +# replay input. Hardcoded paths, skipped if missing (they are run # artifacts, not repo assets). The ledger's stable home is the run dir -# (regenerated byte-identical via run_pilot_from_live_tape); the original -# L1 worktree path is a fallback for machines that still have it. +# (regenerated byte-identical via run_pilot_from_live_tape); the older +# checkout path is a fallback for machines that still have it. L1_TAPE = MAIN_CHECKOUT / "runs/live/20261010T071750Z/live_tape.jsonl" _L1_EVENT_LOG_STABLE = L1_TAPE.parent / "event_log.jsonl" -_L1_EVENT_LOG_LEGACY = Path("/Users/sushantgautam/Documents/verdi-L1/runs/live/event_log.jsonl") +_L1_EVENT_LOG_LEGACY = Path(__file__).resolve().parents[3] / "verdi-L1" / "runs" / "live" / "event_log.jsonl" L1_EVENT_LOG = _L1_EVENT_LOG_STABLE if _L1_EVENT_LOG_STABLE.is_file() else _L1_EVENT_LOG_LEGACY L1_FINAL_STATE_HASH = "fa30e29786877c6534013a2f6a2683a6d19e1cb136a66830d35cd9a8defdd582" @@ -297,7 +297,7 @@ def test_live_run_stamps_simpleaudit_version_on_manifest(tmp_path, monkeypatch) def test_live_adapter_counts_calls_and_delegates(tmp_path) -> None: - from verdi.pilot.live import LiveModelAdapter + from verdi_agents.reference_run.live import LiveModelAdapter tape = _tape_in_dispatch_order() fake = _make_fake(tape) @@ -313,35 +313,35 @@ def test_live_adapter_counts_calls_and_delegates(tmp_path) -> None: # --------------------------------------------------------------------------- # -# ADR-010 first-class replay: the captured L1 live tape must reproduce the +# Tape replay: the captured live-model tape must reproduce the # run's MODEL-DRIVEN ledger byte-for-byte, offline, with no network. # --------------------------------------------------------------------------- # def _require_l1_artifacts() -> None: - """Skip guard: the L1 live-run artifacts are run artifacts, not repo + """Skip guard: the recorded live-run files are run artifacts, not repo assets; skip (not fail) when the artifacts are absent from this machine. """ if not L1_TAPE.is_file(): pytest.skip( - f"L1 captured tape not present on this machine: {L1_TAPE} " - "(ADR-010 replay provenance is a run artifact)" + f"captured live tape not present on this machine: {L1_TAPE} " + "(it is a run artifact, not part of the repo)" ) if not L1_EVENT_LOG.is_file(): pytest.skip( - f"original L1 event log not present on this machine: {L1_EVENT_LOG} " - "(ADR-010 replay provenance is a run artifact)" + f"original live event log not present on this machine: {L1_EVENT_LOG} " + "(it is a run artifact, not part of the repo)" ) def test_tape_replay_reproduces_live_ledger_byte_for_byte(tmp_path) -> None: - """Offline replay of the real captured L1 live tape (no network) + """Offline replay of the real captured live tape (no network) reproduces the model-driven ledger: 126 events, the same final state hash, and a 126/126 event_hash chain equal to the ORIGINAL live run's event_log.jsonl — not just the scaffold-invariant state hash. """ _require_l1_artifacts() - from verdi.pilot.live import run_pilot_from_live_tape + from verdi_agents.reference_run.live import run_pilot_from_live_tape pilot = run_pilot_from_live_tape(L1_TAPE, repo_root=MAIN_CHECKOUT, out_dir=tmp_path) @@ -380,11 +380,11 @@ def test_tape_replay_reproduces_live_ledger_byte_for_byte(tmp_path) -> None: def test_tape_replay_caller_reconstructs_model_result() -> None: """TapeReplayCaller exposes last_result/last_prompt from a tape line and pops responses in seq order; missing ids fall back to deterministic - seq-derived values (pre-F4 tapes). + seq-derived values (older tapes without ids). """ import asyncio - from verdi.pilot.live import TapeReplayCaller + from verdi_agents.reference_run.live import TapeReplayCaller rows = [ {"seq": 0, "response_raw": '{"a": 1}', "model": "Qwen3.8-27B", @@ -414,7 +414,7 @@ async def pop(): r1 = asyncio.run(pop()) assert r1.content == '{"a": 2}' - # missing ids fall back deterministically from seq (pre-F4 tapes) + # missing ids fall back deterministically from seq (older tapes) assert r1.request_id == "replay-req-1" assert r1.trace_id == "replay-trace-1" assert r1.reasoning_content == "" diff --git a/tests/pilot/test_pilot.py b/tests/reference_run/test_reference_run.py similarity index 92% rename from tests/pilot/test_pilot.py rename to tests/reference_run/test_reference_run.py index 6bfecd6..11374e4 100644 --- a/tests/pilot/test_pilot.py +++ b/tests/reference_run/test_reference_run.py @@ -1,6 +1,6 @@ -"""Cross-module pilot integration tests (P1-I, G2/G3). +"""End-to-end tests of the reference run, wiring all core modules together. -These wire every frozen-contract module together over the recorded-output +These run every core module together over the recorded-output pilot_v1 scenario and assert the non-negotiable invariants: * deterministic replay reproduces the substantive state hash; * the event ledger verifies and every attempted violation is logged; @@ -16,7 +16,7 @@ import pytest -from verdi.pilot.driver import ( +from verdi_agents.reference_run.driver import ( PROBE_TICKS, PilotDriver, probe_response_for, @@ -31,7 +31,7 @@ def pilot(): def test_two_runs_are_byte_identical_in_state(pilot) -> None: - """Running the pilot twice from the same scenario yields the same hash.""" + """Running the reference run twice from the same scenario yields the same hash.""" first_hash = pilot.final_state.state_hash again = PilotDriver(REPO_ROOT).run() assert again.final_state.state_hash == first_hash @@ -43,7 +43,7 @@ def test_event_log_verifies(pilot) -> None: def test_replay_reproduces_full_ledger(pilot) -> None: - """G3: folding the FULL recorded ledger tape (all 126 action AND + """Folding the FULL recorded ledger tape (all 126 action AND governance steps, in ledger order) through a fresh reducer + log reproduces the live run's final state hash AND a byte-identical JSONL ledger. No bookkeeping force-copies are needed: the reducer @@ -88,7 +88,7 @@ def test_governance_lifecycle_present(pilot) -> None: assert "amendment_enacted" in kinds assert "sanction_imposed" in kinds assert "appeal_filed" in kinds - # The appeal was adjudicated (dismissed in this pilot), so a dismissal event + # The appeal was adjudicated (dismissed in this run), so a dismissal event # is present; an upheld appeal would instead emit appeal_upheld. assert "appeal_dismissed" in kinds or "appeal_upheld" in kinds @@ -99,7 +99,7 @@ def test_three_assemblies_ran(pilot) -> None: def test_probes_cover_required_ticks_and_channels(pilot) -> None: - from verdi.pilot.driver import CHANNELS + from verdi_agents.reference_run.driver import CHANNELS seen_ticks = {p.tick for p in pilot.probe_results} for t in PROBE_TICKS: @@ -147,7 +147,7 @@ def test_welfare_keyed_by_lifecycle_tick(pilot) -> None: """Regression (re-review note A): welfare/inequality metrics must be keyed by the LIFECYCLE tick of each snapshot (0 + SNAPSHOT_TICKS), NOT the reducer's cumulative transition counter (which is 126 in this run).""" - from verdi.pilot.driver import SNAPSHOT_TICKS + from verdi_agents.reference_run.driver import SNAPSHOT_TICKS expected = {str(0), *(str(t) for t in SNAPSHOT_TICKS)} assert set(pilot.metrics["welfare_food"]["gini_by_tick"].keys()) == expected diff --git a/tests/scheduler/conftest.py b/tests/scheduler/conftest.py index 695606d..2959806 100644 --- a/tests/scheduler/conftest.py +++ b/tests/scheduler/conftest.py @@ -3,7 +3,7 @@ import pytest -from verdi.schemas import AgentState, Task, ValueProfile, WorldState +from verdi_agents.schemas import AgentState, Task, ValueProfile, WorldState def make_agent(agent_id: str, *, idle_ticks: int = 0, under_sanction: bool = False) -> AgentState: diff --git a/tests/scheduler/fixtures_stress.py b/tests/scheduler/fixtures_stress.py index 0eaeea0..141b65c 100644 --- a/tests/scheduler/fixtures_stress.py +++ b/tests/scheduler/fixtures_stress.py @@ -1,11 +1,11 @@ -"""Deterministic task/world generators for scheduler stress tests (WS-7). +"""Deterministic task/world generators for scheduler stress tests. Test-support only. Reuses ``make_agent`` / ``make_task`` from ``conftest.py`` so Task / AgentState construction stays identical to the unit tests. """ from __future__ import annotations -from verdi.schemas import AgentState, WorldState +from verdi_agents.schemas import AgentState, WorldState from .conftest import make_agent, make_task diff --git a/tests/scheduler/test_adaptive.py b/tests/scheduler/test_adaptive.py index 10a087b..506dd29 100644 --- a/tests/scheduler/test_adaptive.py +++ b/tests/scheduler/test_adaptive.py @@ -3,7 +3,7 @@ import pytest -from verdi.scheduler import ( +from verdi_agents.scheduler import ( ADAPTIVE_PROMPT_VERSION, AdaptiveScheduleError, AdaptiveScheduler, diff --git a/tests/scheduler/test_seeded.py b/tests/scheduler/test_seeded.py index 68baf5b..42dc6b5 100644 --- a/tests/scheduler/test_seeded.py +++ b/tests/scheduler/test_seeded.py @@ -1,4 +1,4 @@ -from verdi.scheduler import SeededScheduler +from verdi_agents.scheduler import SeededScheduler from .conftest import make_agent, make_task diff --git a/tests/scheduler/test_stress.py b/tests/scheduler/test_stress.py index 0d405d2..ab3b272 100644 --- a/tests/scheduler/test_stress.py +++ b/tests/scheduler/test_stress.py @@ -1,4 +1,4 @@ -"""Scheduler stress fixture (WS-7): SeededScheduler stability under scale. +"""Scheduler stress tests: SeededScheduler stability under scale. Pin, with deliberately adversarial (colliding-key) loads: 1. byte-identical determinism across scheduler instances and run orderings; @@ -7,13 +7,13 @@ 4. scoped world-state sensitivity (a single agent's state moves only its own tasks, and the unperturbed world always yields the unperturbed order). -Test-only workstream: source bugs are findings, not fixes. +These tests only check behavior; they do not change the scheduler. """ from __future__ import annotations import time -from verdi.scheduler import SeededScheduler +from verdi_agents.scheduler import SeededScheduler from .conftest import make_agent, make_task from .fixtures_stress import ( diff --git a/tests/society/test_audit_regressions.py b/tests/society/test_audit_regressions.py index 9ac222b..8274c22 100644 --- a/tests/society/test_audit_regressions.py +++ b/tests/society/test_audit_regressions.py @@ -1,12 +1,12 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import asyncio import json from types import SimpleNamespace -from verdi.society.memory import AgentMemory -from verdi.society.tasks import _parse_orchestrator_response, generate_llm_tape +from verdi_agents.society.memory import AgentMemory +from verdi_agents.society.tasks import _parse_orchestrator_response, generate_llm_tape from .test_tasks import make_agent, make_state @@ -63,7 +63,7 @@ def test_consolidated_summary_excludes_recent_entries(): def test_watchdog_counts_in_process_calls_during_long_batches(tmp_path, monkeypatch): # Reflection/consolidation batches write model_calls.jsonl only at the # end; completed calls must still reset the stall deadline. - import verdi.society.live as live_module + import verdi_agents.society.live as live_module (tmp_path / "events.jsonl").write_text("") monkeypatch.setattr(live_module, "_HEARTBEAT_INTERVAL_SECONDS", 0.05) @@ -85,8 +85,8 @@ async def scenario(): def test_prompt_shows_lifecycle_tick_not_ledger_counter(): - from verdi.schemas import Task - from verdi.society.prompting import build_social_prompt_messages + from verdi_agents.schemas import Task + from verdi_agents.society.prompting import build_social_prompt_messages state = make_state({"a1": make_agent("a1")}, tick=104) task = Task(task_id="soc-t07-00", actor_id="a1", action_type="produce", @@ -98,7 +98,7 @@ def test_prompt_shows_lifecycle_tick_not_ledger_counter(): def test_metrics_count_only_accepted_sanctions_and_filed_appeals(): - from verdi.society.campaign import _compute_metrics + from verdi_agents.society.campaign import _compute_metrics result = SimpleNamespace( probes=[], constitution_versions=[], quarantined=[], assembly_quarantined={}, @@ -118,7 +118,7 @@ def test_metrics_count_only_accepted_sanctions_and_filed_appeals(): def test_inner_life_prompt_uses_lifecycle_tick(): - from verdi.society.inner_life import build_inner_life_prompt_messages + from verdi_agents.society.inner_life import build_inner_life_prompt_messages state = make_state({"a1": make_agent("a1")}, tick=104) user = build_inner_life_prompt_messages("a1", state, AgentMemory(), tick=7)[1]["content"] @@ -126,7 +126,7 @@ def test_inner_life_prompt_uses_lifecycle_tick(): def test_probe_subjects_follow_send_order_not_completion_order(): - from verdi.society.campaign import _RecordingCaller + from verdi_agents.society.campaign import _RecordingCaller class _Slow: async def generate_structured(self, messages, **kw): diff --git a/tests/society/test_campaign.py b/tests/society/test_campaign.py index c2b56e3..7a8a453 100644 --- a/tests/society/test_campaign.py +++ b/tests/society/test_campaign.py @@ -1,9 +1,9 @@ -"""Offline (fake ModelCaller) tests for the Society V2 campaign orchestrator. +"""Offline (fake ModelCaller) tests for the society campaign orchestrator. These exercise the real deterministic core — reducer, hash-chained ledger, validator, sanction/appeal engine, probe leak guard, task tape, private memory, and the real assembly — with a scripted fake model caller. No network, no LLM. -The G3 replay-evidence claims here are about the deterministic ledger, not +The replay-evidence claims here are about the deterministic ledger, not about the live model's output distribution. """ from __future__ import annotations @@ -16,11 +16,11 @@ import pytest -from verdi.agents.client import ModelResult -from verdi.schemas import ProposedAction -from verdi.society.campaign import SocietyCampaign -from verdi.world.log import EventLog -from verdi.world.validate import validate_action +from verdi_agents.agents.client import ModelResult +from verdi_agents.schemas import ProposedAction +from verdi_agents.society.campaign import SocietyCampaign +from verdi_agents.world.log import EventLog +from verdi_agents.world.validate import validate_action # --------------------------------------------------------------------------- # @@ -130,7 +130,7 @@ def test_full_offline_run_completes_and_replays(scenario, tmp_path): assert any(s.kind == "rights_change" for s in res.tape) assert len(res.events) > 0 assert res.final_state_hash and len(res.final_state_hash) == 64 - # G3: recorded-output replay reproduces state hash AND byte-identical ledger. + # Recorded-output replay reproduces state hash AND byte-identical ledger. assert res.replay_matches is True assert res.replay_log_identical is True # ledger hash chain verifies (run() calls log.verify; assert non-empty chain). @@ -143,7 +143,7 @@ def test_single_file_scenario_defines_governance(tmp_path): """Tranche 12k: the scenario JSON is the complete definition — a custom constitution/law block in the file drives the run, and a file WITHOUT those blocks gets defaults identical to the historical hardcoded values.""" - from verdi.analysis.scenario import build_t0_state + from verdi_agents.analysis.scenario import build_t0_state base = { "scenario_id": "t", "world_id": "t", "run_seed": 1, @@ -187,7 +187,7 @@ def test_minimal_yaml_scenario_runs_end_to_end_offline(tmp_path): confirm the engine derives governance + probe schedule from defaults. Uses a caller that proposes+votes for any amendment so the assembly enacts (quorum 0.5 / threshold 0.67 are met with 3 voters).""" - from verdi.analysis.scenario import build_t0_state, load_scenario + from verdi_agents.analysis.scenario import build_t0_state, load_scenario scenario = load_scenario("tests/fixtures/scenarios/minimal.yaml") # the minimal file intentionally omits the derivable governance/lifecycle @@ -295,7 +295,7 @@ def test_load_scenario_yaml_and_json_parity(tmp_path): """Tranche 12l: load_scenario reads .json and .yml identically (YAML is a JSON superset), and a scenario that omits the lifecycle block gets an auto-derived probe schedule (baseline@0, pre/post assembly, final).""" - from verdi.analysis.scenario import load_scenario + from verdi_agents.analysis.scenario import load_scenario # JSON path unchanged json_path = Path("tests/fixtures/scenarios/society_v2_quick_5x50.json") @@ -526,7 +526,7 @@ def test_mixed_assembly_ledgers_rejected_sibling_and_replays(scenario, tmp_path) assert rejected[0].payload.get("invalid_reasons"), "rejection must carry invalid_reasons" # the enacted proposal bumped the constitution. assert res.state.constitution_version != res.constitution_versions[0] - # G3: both governance outcomes replay to an identical state hash and ledger. + # Both governance outcomes replay to an identical state hash and ledger. assert res.replay_matches is True assert res.replay_log_identical is True @@ -633,8 +633,8 @@ def test_format_failure_is_rejected_without_sanction(scenario, tmp_path): def test_every_call_logged_action_reflection_probe_assembly(scenario, tmp_path): - """N1 (protocol §Lane C2/throughput): EVERY model call appends one row. - Stage A scripted run with enact=False: 12x6 action + 6 reflection + """EVERY model call appends one row to the call log. + Small scripted run with enact=False: 12x6 action + 6 reflection (tick 11) + 90 probe (5 phases x 6 agents x 3 channels) calls, and zero assembly calls (every voter abstains -> no proposals -> no votes). The enacted variant adds 4 deliberation + 24 vote rows (see the @@ -692,7 +692,7 @@ def test_rejected_assembly_is_ledgered_and_replays(scenario, tmp_path): # assembly calls (deliberation + votes) are logged with voter subjects. asm = [c for c in res.model_calls if c.purpose == "assembly"] assert {c.subject for c in asm} == {a["agent_id"] for a in scenario["agents"]} - # G3: rejections replay to an identical state hash and ledger. + # Rejections replay to an identical state hash and ledger. assert res.replay_matches is True assert res.replay_log_identical is True @@ -708,8 +708,8 @@ def test_two_same_tick_withdraws_do_not_double_spend(): against the start-of-tick snapshot (5.0 available, 4.0 each), but the second is re-validated after the first is committed (1.0 left). """ - from verdi.analysis.scenario import build_t0_state - from verdi.world.reducer import WorldReducer + from verdi_agents.analysis.scenario import build_t0_state + from verdi_agents.world.reducer import WorldReducer sc = { "scenario_id": "t", "world_id": "t", "run_seed": 1, @@ -756,8 +756,8 @@ def test_rights_change_flips_world_state_and_blocks_sanctioned_agent(): event_cursor advance, but it DOES change the canonical hash) and the validator + task-tape eligibility actually see it. """ - from verdi.analysis.scenario import build_t0_state - from verdi.world.reducer import WorldReducer + from verdi_agents.analysis.scenario import build_t0_state + from verdi_agents.world.reducer import WorldReducer sc = { "scenario_id": "t", "world_id": "t", "run_seed": 1, @@ -830,9 +830,9 @@ def test_no_probe_leak_into_agent_surfaces(scenario, tmp_path): surface the agents can see (here: pre-seeded into an agent memory).""" campaign = SocietyCampaign(scenario=scenario, run_dir=tmp_path, caller=ScriptedCaller(), run_seed=20261011, concurrency=4) - from verdi.analysis.scenario import build_t0_state - from verdi.measurement.probes import ProbeBank - from verdi.society.memory import AgentMemory + from verdi_agents.analysis.scenario import build_t0_state + from verdi_agents.measurement.probes import ProbeBank + from verdi_agents.society.memory import AgentMemory sc = build_t0_state(scenario) bank = ProbeBank("verdi.probe-bank/1", "verdi.social-instrument/1", rubric_id="verdi.social", pool_size=4) item = bank.draw("s01", 0, "declared", rng_seed=20261011) @@ -846,8 +846,8 @@ def test_deterministic_tape_and_validation(scenario, tmp_path): """Same seed + snapshot -> identical tape; validator rejects overdraft.""" import random - from verdi.analysis.scenario import build_t0_state - from verdi.society.tasks import generate_tick_tape + from verdi_agents.analysis.scenario import build_t0_state + from verdi_agents.society.tasks import generate_tick_tape sc = build_t0_state(scenario) t1 = generate_tick_tape(0, sc, random.Random(), tape_seed="soc-v2:1") t2 = generate_tick_tape(0, sc, random.Random(), tape_seed="soc-v2:1") @@ -886,8 +886,8 @@ def test_parallel_assembly_does_not_change_outcomes(scenario, tmp_path, monkeypa asserts the parallel path actually overlaps model calls.""" import copy - from verdi.governance.assembled import _run_sequential - from verdi.society import campaign as campaign_module + from verdi_agents.governance.assembly import _run_sequential + from verdi_agents.society import campaign as campaign_module class ConcurrencyProbeCaller(ScriptedCaller): def __init__(self, *a, **kw) -> None: diff --git a/tests/society/test_stageC_offline_scale.py b/tests/society/test_campaign_offline_scale.py similarity index 93% rename from tests/society/test_stageC_offline_scale.py rename to tests/society/test_campaign_offline_scale.py index 91dad13..81873e5 100644 --- a/tests/society/test_stageC_offline_scale.py +++ b/tests/society/test_campaign_offline_scale.py @@ -1,11 +1,9 @@ -"""Society V2 Stage C OFFLINE scale tests (mock caller): the 24x60 and 48x100 +"""Offline scale tests (mock caller) for the 24x60 and 48x100 society scenario rungs of the scale ladder, three/five real (model-voted offline) assemblies, ring topology, 1 adversary each (c13 / d25). -Purpose (Stage C scale-jump gate, docs/STAGE_C_PREREGISTRATION.md E1-E4): -prove the verified Stage B machinery is *mechanically valid* at 2x and 4x -Stage B scale BEFORE spending live API (C-2 live is additionally gated on the -C-1 live run passing). Runs the real generated scenarios through the full +Purpose: prove the verified 12-agent machinery is *mechanically valid* at 2x +and 4x that scale BEFORE spending live API calls. Runs the real generated scenarios through the full SocietyCampaign engine with deterministic scripted agents. Pins per rung: @@ -27,9 +25,9 @@ import pytest -from verdi.agents.client import ModelResult -from verdi.society.campaign import SocietyCampaign -from verdi.world.log import EventLog +from verdi_agents.agents.client import ModelResult +from verdi_agents.society.campaign import SocietyCampaign +from verdi_agents.world.log import EventLog SCENARIOS_DIR = Path(__file__).resolve().parents[2] / "tests/fixtures/scenarios" @@ -124,7 +122,7 @@ def _run(scenario: dict, tmp_path: Path) -> Any: # (scenario file, n agents, total ticks, assemblies, adversary, probe ticks) RUNGS = [ ("society_v2_stageC1.json", 24, 60, [15, 30, 45], "c13", 9), - # C-2 reflection lands at 33 (total//3), adding two more probe ticks + # The 48-agent run's reflection lands at 33 (total//3), adding two more probe ticks # (baseline, 4 pre-meeting, 5 assembly, reflection, post = 13). ("society_v2_stageC2.json", 48, 100, [20, 40, 60, 80, 95], "d25", 13), ] diff --git a/tests/society/test_stageB.py b/tests/society/test_campaign_scenarios.py similarity index 95% rename from tests/society/test_stageB.py rename to tests/society/test_campaign_scenarios.py index 29a89c5..78b4e71 100644 --- a/tests/society/test_stageB.py +++ b/tests/society/test_campaign_scenarios.py @@ -1,7 +1,7 @@ -"""Society V2 Stage B offline tests (mock caller): 12 agents, 30 ticks, +"""Offline tests (mock caller) for the 12-agent, 30-tick society scenario: three real (model-voted offline) assemblies, reflection at t11, adversary b05. -These pin the Stage B scenario's structural invariants OFFLINE so the live +These pin the scenario's structural invariants OFFLINE so the live run can be compared against a deterministic expectation before any endpoint cost is spent. """ @@ -12,9 +12,9 @@ from pathlib import Path from typing import Any -from verdi.agents.client import ModelResult -from verdi.society.campaign import SocietyCampaign -from verdi.world.log import EventLog +from verdi_agents.agents.client import ModelResult +from verdi_agents.society.campaign import SocietyCampaign +from verdi_agents.world.log import EventLog SCENARIO_PATH = Path(__file__).resolve().parents[2] / "tests/fixtures/scenarios/society_v2_stageB.json" AGENTS = tuple(f"b{i:02d}" for i in range(1, 13)) @@ -25,7 +25,7 @@ def _scenario() -> dict: class StageBScriptedCaller: - """Deterministic fake ModelCaller for Stage B agent IDs. + """Deterministic fake ModelCaller for this scenario's agent IDs. Adversary b05 attempts an overdraft transfer on its first action (rejected by the validator); everyone else emits a signal. @@ -141,7 +141,7 @@ def test_stageB_offline_run_completes_and_replays(tmp_path): b05_sanction_tick = [s for s in sanctioned_tick_steps if s.action.actor_id == "b05"] assert len(b05_sanction_tick) == 1, "b05 must act during his sanctioned tick (not be suspended)" - # G3: byte-identical replay + hash chain. + # Replay is byte-identical and the hash chain holds. assert res.replay_matches is True assert res.replay_log_identical is True log = EventLog.load_jsonl(tmp_path / "events.jsonl") diff --git a/tests/society/test_economy_campaign.py b/tests/society/test_economy_campaign.py index 50c7efe..2d29563 100644 --- a/tests/society/test_economy_campaign.py +++ b/tests/society/test_economy_campaign.py @@ -1,4 +1,4 @@ -"""P0-B: offline (fake ModelCaller) campaign tests for the constrained +"""Offline (fake ModelCaller) campaign tests for the constrained economy + objective task-success oracle. Shows, with a fully scripted caller and no LLM: @@ -18,9 +18,9 @@ from pathlib import Path from typing import Any -from verdi.agents.client import ModelResult -from verdi.analysis.scenario import load_scenario -from verdi.society.campaign import SocietyCampaign +from verdi_agents.agents.client import ModelResult +from verdi_agents.analysis.scenario import load_scenario +from verdi_agents.society.campaign import SocietyCampaign # --------------------------------------------------------------------------- # @@ -121,7 +121,7 @@ def _scenario() -> dict[str, Any]: sc = load_scenario("tests/fixtures/scenarios/institutional_recovery_smoke.yaml") # The smoke file carries min_commons_reserve {food: 12.0}: withdrawals are # constitutionally floored at 12. This test drives deposits, never - # withdrawals, but drop the floor so the run exercises ONLY the P0-B + # withdrawals, but drop the floor so the run exercises ONLY the economy # machinery (cap/cost/consumption/objectives) in isolation. sc["constitution"]["rules"].pop("min_commons_reserve", None) return sc diff --git a/tests/society/test_live.py b/tests/society/test_live.py index d1108ae..ed052c0 100644 --- a/tests/society/test_live.py +++ b/tests/society/test_live.py @@ -1,4 +1,4 @@ -"""Offline tests for the Society V2 live entrypoint (mocked transport). +"""Offline tests for the live society-run entrypoint (mocked transport). No network: the pooled ModelClient runs against httpx.MockTransport. Verifies env resolution, the pre-run smoke call, end-to-end live wiring, @@ -13,9 +13,9 @@ import httpx import pytest -from verdi.agents.client import ModelClient -from verdi.config import ModelEndpointConfig -from verdi.society import live as live_module +from verdi_agents.agents.client import ModelClient +from verdi_agents.config import ModelEndpointConfig +from verdi_agents.society import live as live_module SCENARIO = Path("tests/fixtures/scenarios/society_v2_stageA.json") @@ -136,7 +136,7 @@ def factory(config: ModelEndpointConfig, **kwargs: Any) -> ModelClient: # free of key material under the broad patterns. for path in run_dir.rglob("*"): if path.is_file() and path != report: - from verdi.audit.judge_live import _assert_no_key_material + from verdi_agents.audit.judge import _assert_no_key_material _assert_no_key_material(path.read_text(encoding="utf-8", errors="replace"), str(path)) # The derived report embeds the plotly.js open-source bundle (which trips # the broad sk-*/api_key= patterns), so it is guarded PRECISELY: it must diff --git a/tests/society/test_memory.py b/tests/society/test_memory.py index 514aa67..454b8d1 100644 --- a/tests/society/test_memory.py +++ b/tests/society/test_memory.py @@ -1,9 +1,9 @@ -"""Tests for src/verdi/society/memory.py.""" +"""Tests for src/verdi_agents/society/memory.py.""" from __future__ import annotations import pytest -from verdi.society import AgentMemory +from verdi_agents.society import AgentMemory def test_version_bumps_on_appends(): @@ -50,7 +50,7 @@ def test_per_agent_privacy(): def test_consolidation_triggers_when_window_overflows(): """Consolidation triggers when overflow >= 4 entries (batched, not every tick).""" - from verdi.society.memory import _RECENT_WINDOW + from verdi_agents.society.memory import _RECENT_WINDOW mem = AgentMemory() # Below threshold: window + 3 overflow → no consolidation yet for i in range(_RECENT_WINDOW + 3): @@ -75,7 +75,7 @@ def test_consolidation_triggers_when_window_overflows(): def test_consolidation_preserves_recent_content(): """After consolidation, recent entries are untouched.""" - from verdi.society.memory import _RECENT_WINDOW + from verdi_agents.society.memory import _RECENT_WINDOW mem = AgentMemory() for i in range(_RECENT_WINDOW + 2): mem.append_reflection("a1", f"memory {i}") diff --git a/tests/society/test_prompting.py b/tests/society/test_prompting.py index d5d1f3f..001cec3 100644 --- a/tests/society/test_prompting.py +++ b/tests/society/test_prompting.py @@ -1,13 +1,13 @@ -"""Tests for src/verdi/society/prompting.py. +"""Tests for src/verdi_agents/society/prompting.py. -Contains the acceptance-critical prompt-observable feedback test -(docs/SOCIETY_V2_PROTOCOL.md, section 3): a later agent's prompt MUST +Contains the acceptance-critical prompt-observable feedback test: +a later agent's prompt MUST change when an earlier committed event changes. """ from __future__ import annotations -from verdi.schemas import AgentState, RightsState, Task, ValueProfile, WorldState -from verdi.society import SOCIAL_PROMPT_VERSION, AgentMemory, build_social_prompt +from verdi_agents.schemas import AgentState, RightsState, Task, ValueProfile, WorldState +from verdi_agents.society import SOCIAL_PROMPT_VERSION, AgentMemory, build_social_prompt _VALUES = ValueProfile( integrity=0.5, fairness=0.5, cooperation=0.5, autonomy=0.5, solidarity=0.5 diff --git a/tests/society/test_tasks.py b/tests/society/test_tasks.py index 9049868..f46bb50 100644 --- a/tests/society/test_tasks.py +++ b/tests/society/test_tasks.py @@ -1,11 +1,11 @@ -"""Tests for src/verdi/society/tasks.py.""" +"""Tests for src/verdi_agents/society/tasks.py.""" from __future__ import annotations import random -from verdi.schemas import AgentState, ValueProfile, WorldState -from verdi.society import MUTATING_KINDS, generate_tick_tape -from verdi.society.memory import ADVERSARY_MARKER +from verdi_agents.schemas import AgentState, ValueProfile, WorldState +from verdi_agents.society import MUTATING_KINDS, generate_tick_tape +from verdi_agents.society.memory import ADVERSARY_MARKER _BASE_PERSONA = "a cooperative citizen" @@ -163,7 +163,7 @@ def test_unsanctioned_tape_is_unchanged_shape(): # Byte-identity guard: a state with NO sanctioned agents yields exactly # one task per agent (the pre-change behavior). The no-sanction branch of # generate_tick_tape is untouched by the sanctioned-tape change, so frozen - # pilot L1 / G5 placebo tapes (no sanctioned agents) reproduce identically. + # reference-run and placebo tapes (no sanctioned agents) reproduce identically. state = make_state({a: make_agent(a) for a in ("a1", "a2", "a3")}, tick=1) tape = generate_tick_tape(1, state, random.Random(0), tape_seed=9) assert sorted(t.actor_id for t in tape) == ["a1", "a2", "a3"] diff --git a/tests/society/test_watchdog_calibrated.py b/tests/society/test_watchdog_calibrated.py index c6575fa..bd72230 100644 --- a/tests/society/test_watchdog_calibrated.py +++ b/tests/society/test_watchdog_calibrated.py @@ -1,15 +1,15 @@ -"""Tranche 12d (review F1) regression tests for the live stall watchdog. +"""Regression tests for the live stall watchdog. -F1 (HIGH, independent review of feat/society-v2-stage-c): a full assembly +The bug: a full assembly tick appends NOTHING to events.jsonl until its very end, so an -events-only watchdog threshold false-fires on legitimately long ticks. The -live C-1 attempt was killed at t15 (its first assembly) on exactly this +events-only watchdog threshold false-fires on legitimately long ticks. An +earlier 24-agent live run was killed at t15 (its first assembly) on exactly this path. The fix adds per-model-call liveness beats (live._LiveProgressCaller -> progress.jsonl) that reset the watchdog deadline on every COMPLETED call, so the watchdog only trips when no call finishes at all (a genuinely stuck socket). -Tranche 12i: the per-call liveness signal moved from the old +Later, the per-call liveness signal moved from the old progress.jsonl wrapper to the campaign's own streaming model_calls.jsonl writes (one line appended per completed call), so these tests now model that real signal via _ModelCallsCaller. @@ -21,7 +21,7 @@ and the combined-progress watchdog does NOT fire during a long sequential tick while beats keep flowing; * behavior: with the inner caller hanging, the watchdog still fires - (the Tranche 12a bug stays covered). + (the earlier hung-socket bug stays covered). """ from __future__ import annotations @@ -34,14 +34,14 @@ import pytest -from verdi.agents.client import ModelResult -from verdi.society import live as live_module +from verdi_agents.agents.client import ModelResult +from verdi_agents.society import live as live_module SCEN = Path(__file__).resolve().parents[2] / "tests/fixtures/scenarios/society_v2_stageC1.json" -# Observed SimulaChat Qwen3.8-27B latencies from Stage B / C-1 attempts +# Observed SimulaChat Qwen3.8-27B latencies from earlier 12- and 24-agent live runs # (runs/society-v2/stageB-* health.json): mean 3.36-4.65 s/call, -# p95 8.9-11.5 s. Used by the F1 false-fire documentation test. +# p95 8.9-11.5 s. Used by the false-fire documentation test. P95_CALL_SECONDS = 8.9 @@ -49,14 +49,14 @@ def _assembly_tick_lower_bound_calls(n: int) -> int: """Lower bound on model calls in one n-agent assembly tick, SEQUENTIAL. Verified against campaign._run_tick order on an assembly tick (the - pre-Tranche-12f path; the F1 false-fire arithmetic it documents): + sequential path from before the parallel runner; the false-fire arithmetic it documents): 1. probe phases run first: n agents x 3 channels, sequential = 3n. 2. run_deliberation: one call per agent = n. 3. run_voting: one call per (proposal, voter) with >= 1 proposal, all n agents hold voting rights in these scenarios = n. Floor = 5n (n agents, 3 probe channels, >= 1 proposal). - Tranche 12f note: deliberation (n) and voting (>=n) now run through a + Note: deliberation (n) and voting (>=n) now run through a bounded-parallel runner, so their WALL-CLOCK contribution drops to ~ceil(n/concurrency) waves at concurrency >= n; only the 3n probe calls remain sequential. This 5n floor therefore overstates live wall-clock @@ -83,10 +83,10 @@ def test_watchdog_threshold_covers_worst_case_stuck_call(monkeypatch: pytest.Mon monkeypatch.setenv("VERDI_VLLM_API_KEY", "test-key-not-real") monkeypatch.setenv("VERDI_REQUEST_TIMEOUT", "180") monkeypatch.setenv("VERDI_MAX_RETRIES", "5") - from verdi.config import model_endpoint_from_env + from verdi_agents.config import model_endpoint_from_env cfg = model_endpoint_from_env() # httpx total timeout bounds a normally-timed-out stuck call, but the - # Tranche 12a bug is a held-open-never-read socket that httpx's timeout + # earlier hung-socket bug is a held-open-never-read socket that httpx's timeout # does NOT fire on — only the watchdog catches that. So the window must # at least cover a full stuck-call retry sequence plus one heartbeat # interval (and in practice well beyond it, to catch the @@ -101,13 +101,13 @@ def test_watchdog_threshold_covers_worst_case_stuck_call(monkeypatch: pytest.Mon def test_events_only_design_would_false_fire_on_assembly_tick(): - """Documents WHY the events-only threshold was wrong (F1): the worst-case + """Documents WHY the events-only threshold was wrong: the worst-case duration of one n-agent assembly tick at p95 call latency exceeds the - old 600s events-only window. This is the arithmetic that killed the live - C-1 run at t15. (The current design avoids it because per-call beats + old 600s events-only window. This is the arithmetic that killed an earlier + 24-agent live run at t15. (The current design avoids it because per-call beats reset the deadline mid-tick; this test pins the unsafe-arithmetic guard.) - Tranche 12f note: the 5n floor is the SEQUENTIAL wall-clock model. After + Note: the 5n floor is the SEQUENTIAL wall-clock model. After the parallel runner landed, deliberation+voting (~2n calls) complete in ~ceil(n/concurrency) waves, so the live p95 assembly tick is closer to 3n (sequential probes) + ceil(n/concurrency)*p95 — well under the old @@ -116,7 +116,7 @@ def test_events_only_design_would_false_fire_on_assembly_tick(): changed enough that the per-call-beat justification must be re-examined in the other direction. """ - n = len(json.loads(SCEN.read_text())["agents"]) # 24 for C-1 + n = len(json.loads(SCEN.read_text())["agents"]) # 24 for this scenario floor_calls = _assembly_tick_lower_bound_calls(n) # 5n = 120 p95_tick_seconds = floor_calls * P95_CALL_SECONDS assert p95_tick_seconds > 600, ( @@ -127,7 +127,7 @@ def test_events_only_design_would_false_fire_on_assembly_tick(): class _ModelCallsCaller: - """Tranche 12i: mirrors the campaign's streaming write — appends one + """Mirrors the campaign's streaming write — appends one line to model_calls.jsonl per COMPLETED call. Calls whose content is 'WEDGED' hang forever (no line is written for them). This replaces the old _LiveProgressCaller, which is gone: liveness now comes from the @@ -161,7 +161,7 @@ def close(self) -> None: def test_watchdog_masks_single_wedged_call_under_parallelism(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - """Tranche 12f sensitivity boundary, documented honestly. + """Sensitivity boundary under the parallel runner, documented honestly. Under the parallel runner, a SINGLE wedged call (held-open-never-read socket) among OTHER in-flight calls that keep completing is MASKED: the @@ -287,7 +287,7 @@ async def go(): def test_watchdog_still_fires_when_call_hangs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - """Tranche 12a coverage stays: a hung in-flight call (no beats, no + """Regression for the earlier hung-socket bug: a hung in-flight call (no beats, no events) still trips the watchdog with exit code 3.""" monkeypatch.setattr(live_module, "_HEARTBEAT_INTERVAL_SECONDS", 0.05) (tmp_path / "events.jsonl").write_text("") diff --git a/tests/test_cli.py b/tests/test_cli.py index bddb3ee..2eedfb9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,24 +8,42 @@ def test_cli_runs() -> None: env_python = sys.executable result = subprocess.run( - [env_python, "-m", "verdi.cli"], + [env_python, "-m", "verdi_agents.cli"], cwd=ROOT, capture_output=True, text=True, check=False, ) assert result.returncode == 0 - assert "usage: verdi" in result.stdout - for command in ("init", "run", "sweep"): + assert "usage: verdi-agents" in result.stdout + for command in ("init", "run", "sweep", "pair", "compare"): assert command in result.stdout assert "IMPLEMENTATION_PLAN" not in result.stdout def test_run_respects_run_dir_equals_form(monkeypatch) -> None: - import verdi.cli as cli - import verdi.society.live as live + import verdi_agents.cli as cli + import verdi_agents.society.live as live seen: list[list[str]] = [] monkeypatch.setattr(live, "main", lambda argv: seen.append(argv) or 0) assert cli.main(["run", "examples/starter.yaml", "--run-dir=out/mine"]) == 0 assert seen == [["--scenario", "examples/starter.yaml", "--run-dir=out/mine"]] + + +def test_pair_and_compare_accept_a_positional_path(monkeypatch) -> None: + import verdi_agents.analysis.outcome_metrics as outcome_metrics + import verdi_agents.cli as cli + import verdi_agents.experiments.paired_run as paired_run + + seen: list[tuple[str, list[str]]] = [] + monkeypatch.setattr(paired_run, "main", lambda argv: seen.append(("pair", argv)) or 0) + monkeypatch.setattr(outcome_metrics, "main", lambda argv: seen.append(("compare", argv)) or 0) + assert cli.main(["pair", "examples/paired.yaml", "--seeds", "1,2"]) == 0 + assert cli.main(["compare", "runs/p"]) == 0 + assert cli.main(["compare", "--pair-root", "runs/q"]) == 0 + assert seen == [ + ("pair", ["--config", "examples/paired.yaml", "--seeds", "1,2"]), + ("compare", ["--pair-root", "runs/p"]), + ("compare", ["--pair-root", "runs/q"]), + ] diff --git a/tests/test_config.py b/tests/test_config.py index 99f815c..e1d6630 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ -from verdi.config import model_endpoint_from_env +from verdi_agents.config import model_endpoint_from_env def test_config_from_env(monkeypatch) -> None: diff --git a/tests/test_example_scenarios.py b/tests/test_example_scenarios.py index 888dc08..005d5e6 100644 --- a/tests/test_example_scenarios.py +++ b/tests/test_example_scenarios.py @@ -10,9 +10,9 @@ import pytest -from verdi.agents.client import ModelResult -from verdi.analysis.scenario import build_t0_state, load_scenario -from verdi.society.campaign import SocietyCampaign +from verdi_agents.agents.client import ModelResult +from verdi_agents.analysis.scenario import build_t0_state, load_scenario +from verdi_agents.society.campaign import SocietyCampaign REPO_ROOT = Path(__file__).resolve().parents[1] EXAMPLES = ["starter.yaml", "full.yaml"] diff --git a/tests/test_init.py b/tests/test_init.py index 94507f0..fd6dddb 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,9 +1,9 @@ -"""`verdi init` copies the shipped examples into a working folder.""" +"""`verdi-agents init` copies the shipped examples into a working folder.""" from __future__ import annotations import pytest -from verdi.cli import init, main +from verdi_agents.cli import init, main def test_init_creates_examples_and_env(tmp_path, capsys): @@ -26,7 +26,7 @@ def test_init_refuses_to_overwrite_and_keeps_env(tmp_path): def test_initialized_sweep_validates_from_its_folder(tmp_path, monkeypatch, capsys): - from verdi.experiments import sweep as sw + from verdi_agents.experiments import sweep as sw init(tmp_path) monkeypatch.chdir(tmp_path) @@ -37,7 +37,7 @@ def test_initialized_sweep_validates_from_its_folder(tmp_path, monkeypatch, caps def test_dotenv_in_current_folder_is_loaded(tmp_path, monkeypatch): - from verdi.experiments.g5_judge_live import _load_dotenv_into_environ + from verdi_agents.experiments.pair_judge import _load_dotenv_into_environ (tmp_path / ".env").write_text("VERDI_TEST_CWD_KEY=from-cwd\n") monkeypatch.chdir(tmp_path) @@ -50,8 +50,8 @@ def test_dotenv_in_current_folder_is_loaded(tmp_path, monkeypatch): def test_verdi_run_defaults_run_dir(monkeypatch): - import verdi.society.live as live - from verdi.cli import main as cli_main + import verdi_agents.society.live as live + from verdi_agents.cli import main as cli_main seen = {} monkeypatch.setattr(live, "main", lambda argv: seen.setdefault("argv", argv) and 0) diff --git a/tests/test_researcher_workflow.py b/tests/test_researcher_workflow.py index 0a53ec1..de0c595 100644 --- a/tests/test_researcher_workflow.py +++ b/tests/test_researcher_workflow.py @@ -16,8 +16,8 @@ import copy from pathlib import Path -from verdi.analysis.scenario import load_scenario -from verdi.experiments.paired_run import build_arm_scenarios, load_paired_config +from verdi_agents.analysis.scenario import load_scenario +from verdi_agents.experiments.paired_run import build_arm_scenarios, load_paired_config REPO_ROOT = Path(__file__).resolve().parent.parent PAIRED_CONFIG = REPO_ROOT / "examples" / "paired_institutional_recovery.yaml" @@ -25,8 +25,7 @@ #: The exact one-line command the guide must teach researchers. CLI_COMMAND = ( - "python -m verdi.experiments.paired_run --config " - "examples/paired_institutional_recovery.yaml" + "verdi-agents pair examples/paired_institutional_recovery.yaml" ) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 516ecfe..13c0763 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -1,4 +1,4 @@ -"""Contract tests: fixtures serialize/deserialize and enforce invariants (G0/G1).""" +"""Contract tests: fixtures serialize/deserialize and enforce invariants.""" from __future__ import annotations import json @@ -7,8 +7,8 @@ import pytest from pydantic import ValidationError -from verdi import schemas -from verdi.schemas import ( +from verdi_agents import schemas +from verdi_agents.schemas import ( Appeal, ConstitutionVersion, ExperimentManifest, diff --git a/tests/world/helpers.py b/tests/world/helpers.py index 009c5c5..07d21da 100644 --- a/tests/world/helpers.py +++ b/tests/world/helpers.py @@ -1,10 +1,10 @@ -"""Shared builders for world-engine tests (deterministic, no fixtures beyond G0).""" +"""Shared builders for world-engine tests (deterministic, base-law fixtures only).""" from __future__ import annotations import json from pathlib import Path -from verdi.schemas import AgentState, ProposedAction, ValidationResult, WorldState +from verdi_agents.schemas import AgentState, ProposedAction, ValidationResult, WorldState FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" diff --git a/tests/world/test_audit_regressions.py b/tests/world/test_audit_regressions.py index 44e1307..1d67279 100644 --- a/tests/world/test_audit_regressions.py +++ b/tests/world/test_audit_regressions.py @@ -1,11 +1,11 @@ -"""Regression tests for defects found in the 2026-11 system audit (AUDIT.md).""" +"""Regression tests for previously fixed bugs.""" from __future__ import annotations import pytest -from verdi.schemas import AgentState, ProposedAction, ValueProfile, WorldState -from verdi.world.reducer import InvariantViolationError, WorldReducer -from verdi.world.validate import validate_action +from verdi_agents.schemas import AgentState, ProposedAction, ValueProfile, WorldState +from verdi_agents.world.reducer import InvariantViolationError, WorldReducer +from verdi_agents.world.validate import validate_action _VALUES = ValueProfile( integrity=0.8, fairness=0.9, cooperation=0.7, autonomy=0.5, solidarity=0.85 @@ -56,9 +56,9 @@ def test_non_string_resource_is_rejected_not_crashing(kind, args): def test_branch_from_a_prefix_ending_in_a_rejected_event(): from tests.world.helpers import accepted, action, initial_world, rejected_double_spend - from verdi.world.branch import BranchSpec, branch_from_snapshot - from verdi.world.log import EventLog - from verdi.world.replay import TapeStep, replay + from verdi_agents.world.branch import BranchSpec, branch_from_snapshot + from verdi_agents.world.log import EventLog + from verdi_agents.world.replay import TapeStep, replay log = EventLog() state, _ = replay(log, initial_world(), [ @@ -78,7 +78,7 @@ def test_branch_from_a_prefix_ending_in_a_rejected_event(): def test_snapshot_store_rejects_traversal_and_tampering(tmp_path): from tests.world.helpers import initial_world - from verdi.world.snapshot import SnapshotError, SnapshotStore + from verdi_agents.world.snapshot import SnapshotError, SnapshotStore store = SnapshotStore(tmp_path) digest = store.save(initial_world()) @@ -104,7 +104,7 @@ def test_produce_policy_cap_is_cumulative_per_tick_without_economy(): def test_role_only_consumption_override_is_rejected(): - from verdi.world.economy import parse_economy_spec + from verdi_agents.world.economy import parse_economy_spec with pytest.raises(ValueError, match="consumption_per_tick"): parse_economy_spec({"version": 1, "consumption_per_tick": {"food": 1}, @@ -128,7 +128,7 @@ def test_boolean_amount_is_not_a_quantity(kind, args): def test_duplicate_objective_ids_are_rejected(): - from verdi.world.economy import parse_task_objectives + from verdi_agents.world.economy import parse_task_objectives with pytest.raises(ValueError, match="duplicate objective_id"): parse_task_objectives({"deliveries": [ diff --git a/tests/world/test_branch.py b/tests/world/test_branch.py index f09d878..e55ff5c 100644 --- a/tests/world/test_branch.py +++ b/tests/world/test_branch.py @@ -1,4 +1,4 @@ -"""P2 WS-1: counterfactual snapshot branching tests (guarantees B1-B5).""" +"""Tests for branching a counterfactual run from a saved snapshot (guarantees B1-B5).""" from __future__ import annotations from pathlib import Path @@ -6,16 +6,16 @@ import pytest from tests.world.helpers import accepted, action, initial_world, rejected_double_spend -from verdi.schemas import ProposedAction, ValidationResult -from verdi.world.branch import ( +from verdi_agents.schemas import ProposedAction, ValidationResult +from verdi_agents.world.branch import ( BranchOriginError, BranchSpec, branch_from_snapshot, manifest_for_branch, ) -from verdi.world.log import EventLog -from verdi.world.reducer import WorldReducer -from verdi.world.replay import TapeStep, replay +from verdi_agents.world.log import EventLog +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.replay import TapeStep, replay REPO_ROOT = Path(__file__).resolve().parents[2] @@ -162,7 +162,7 @@ def test_b3_origin_event_count_mismatch(prefix): def test_b3_snapshot_not_ledger_terminal_state_rejected(prefix): - """REGRESSION (P2 review F1): an (origin_state_hash, origin_event_count) + """REGRESSION: an (origin_state_hash, origin_event_count) pair can match a snapshot that is NOT the state the prefix ledger terminates in. Such an inconsistent (snapshot, log) pair must be rejected even though the spec is self-consistent with the (tampered) @@ -249,12 +249,12 @@ def test_two_branches_diverge(prefix): # --------------------------------------------------------------------------- # -# integration: branch from pilot event 28 (first governance event) -# reproduces the pilot final state hash +# integration: branch from reference-run event 28 (first governance event) +# reproduces the reference run's final state hash # --------------------------------------------------------------------------- # @pytest.fixture(scope="module") def pilot(): - from verdi.pilot.driver import PilotDriver + from verdi_agents.reference_run.driver import PilotDriver return PilotDriver(REPO_ROOT).run() @@ -267,7 +267,7 @@ def test_branch_replay_matches_live_ledger(pilot): log = pilot.log n = 28 first_gov = log.events[n] - # event 28 is the first governance event in the pilot ledger + # event 28 is the first governance event in the reference-run ledger assert first_gov.kind == "proposal_submitted" remaining = pilot.full_tape_steps[n:] @@ -292,10 +292,10 @@ def test_branch_replay_matches_live_ledger(pilot): result.log.verify() assert result.log.events[n].prev_hash == prefix_log.last_hash() - # The counterfactual branch reproduces the pilot final state exactly. + # The counterfactual branch reproduces the reference run's final state exactly. assert result.branch_state_hash == WorldReducer.canonical_state_hash( pilot.final_state ) assert result.branch_state_hash == pilot.final_state.state_hash - # and the full branch ledger is byte-identical to the pilot ledger: + # and the full branch ledger is byte-identical to the reference-run ledger: assert result.log.to_jsonl_bytes() == pilot.log.to_jsonl_bytes() diff --git a/tests/world/test_constitutional_policy.py b/tests/world/test_constitutional_policy.py index 41e5323..ae84dcb 100644 --- a/tests/world/test_constitutional_policy.py +++ b/tests/world/test_constitutional_policy.py @@ -3,9 +3,9 @@ import pytest -from verdi.schemas import AgentState, ProposedAction, ValueProfile, WorldState -from verdi.world.policy import validate_enforceable_rules -from verdi.world.validate import validate_action +from verdi_agents.schemas import AgentState, ProposedAction, ValueProfile, WorldState +from verdi_agents.world.policy import validate_enforceable_rules +from verdi_agents.world.validate import validate_action def _state() -> WorldState: diff --git a/tests/world/test_economy.py b/tests/world/test_economy.py index e32cd90..1631784 100644 --- a/tests/world/test_economy.py +++ b/tests/world/test_economy.py @@ -1,4 +1,4 @@ -"""P0-B: unit tests for the opt-in constrained economy + task oracle. +"""Unit tests for the opt-in constrained economy + task oracle. Covers production-limit rejection, production-cost deduction and conservation, consumption non-negativity (no overdraft ever), deficit @@ -10,8 +10,8 @@ import pytest -from verdi.schemas import AgentState, ProposedAction, ValueProfile, WorldState -from verdi.world.economy import ( +from verdi_agents.schemas import AgentState, ProposedAction, ValueProfile, WorldState +from verdi_agents.world.economy import ( apply_consumption, apply_production_deduction, check_delivery, @@ -21,8 +21,8 @@ parse_task_objectives, production_input_cost, ) -from verdi.world.policy import ENFORCEABLE_KEYS, policy_block, validate_enforceable_rules -from verdi.world.validate import validate_action +from verdi_agents.world.policy import ENFORCEABLE_KEYS, policy_block, validate_enforceable_rules +from verdi_agents.world.validate import validate_action _VALUES = ValueProfile( integrity=0.8, fairness=0.9, cooperation=0.7, autonomy=0.5, solidarity=0.85 diff --git a/tests/world/test_log.py b/tests/world/test_log.py index 9ef1074..2b80d4b 100644 --- a/tests/world/test_log.py +++ b/tests/world/test_log.py @@ -6,9 +6,9 @@ import pytest from tests.world.helpers import accepted, action, initial_world, rejected_double_spend -from verdi.schemas import WorldEvent -from verdi.world import EventLog, LogIntegrityError -from verdi.world.reducer import WorldReducer +from verdi_agents.schemas import WorldEvent +from verdi_agents.world import EventLog, LogIntegrityError +from verdi_agents.world.reducer import WorldReducer def _first_events(): diff --git a/tests/world/test_observation_validate.py b/tests/world/test_observation_validate.py index 8785e16..fe25e59 100644 --- a/tests/world/test_observation_validate.py +++ b/tests/world/test_observation_validate.py @@ -1,11 +1,11 @@ -"""Lane W tests: agent_observation + validate_action (pure, deterministic).""" +"""Tests for agent_observation + validate_action (pure, deterministic).""" from __future__ import annotations import pytest -from verdi.schemas import AgentState, ProposedAction, RightsState, ValueProfile, WorldState -from verdi.world.observation import agent_observation -from verdi.world.validate import validate_action +from verdi_agents.schemas import AgentState, ProposedAction, RightsState, ValueProfile, WorldState +from verdi_agents.world.observation import agent_observation +from verdi_agents.world.validate import validate_action def _values() -> ValueProfile: diff --git a/tests/world/test_reducer.py b/tests/world/test_reducer.py index 3bfad72..dfe4b1f 100644 --- a/tests/world/test_reducer.py +++ b/tests/world/test_reducer.py @@ -5,8 +5,8 @@ from pydantic import ValidationError from tests.world.helpers import accepted, action, initial_world, rejected_double_spend -from verdi.world import InvariantViolationError, WorldReducer -from verdi.world.canonical import canonical_sha256 +from verdi_agents.world import InvariantViolationError, WorldReducer +from verdi_agents.world.canonical import canonical_sha256 def test_canonical_hash_is_stable_and_sorted(): diff --git a/tests/world/test_replay.py b/tests/world/test_replay.py index de4c26e..bf2bf78 100644 --- a/tests/world/test_replay.py +++ b/tests/world/test_replay.py @@ -3,7 +3,7 @@ from tests.world.helpers import accepted, action, initial_world, rejected_double_spend -from verdi.world import ( +from verdi_agents.world import ( EventLog, SnapshotStore, TapeStep, @@ -92,7 +92,7 @@ def test_snapshot_roundtrip_byte_identical(tmp_path): loaded = store.load(digest) assert loaded == ws # byte-identical canonical encoding - from verdi.world.canonical import canonical_json + from verdi_agents.world.canonical import canonical_json assert canonical_json(loaded.model_dump(mode="json")) == canonical_json(ws.model_dump(mode="json")) diff --git a/tests/world/test_validate.py b/tests/world/test_validate.py index ce896fa..902e20a 100644 --- a/tests/world/test_validate.py +++ b/tests/world/test_validate.py @@ -1,6 +1,6 @@ -"""Regression tests for the validator/reducer deposit invariant (Tranche 11 follow-up). +"""Regression tests for the validator/reducer deposit invariant. -The live Stage B seed-1 (fix1) run crashed with +An earlier 12-agent live run crashed with ``InvariantViolationError: negative balance: agent=b01 resource=credits value=-2.0`` because ``validate_action`` accepted a ``deposit`` that exceeded the agent's holdings: the deposit/withdraw branch checked only the WORLD pool @@ -9,9 +9,9 @@ """ from __future__ import annotations -from verdi.schemas import AgentState, ProposedAction, ValueProfile, WorldState -from verdi.world.reducer import WorldReducer -from verdi.world.validate import validate_action +from verdi_agents.schemas import AgentState, ProposedAction, ValueProfile, WorldState +from verdi_agents.world.reducer import WorldReducer +from verdi_agents.world.validate import validate_action _VALUES = ValueProfile( integrity=0.8, fairness=0.9, cooperation=0.7, autonomy=0.5, solidarity=0.85