From 8a81d234316ec7d5b2d948a56778ed558235aae5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 16:58:51 -0400 Subject: [PATCH 01/28] Add Meta Muse Wonder's roster entry, price and release date (card WIP) Meta released Muse Wonder on 2026-10-12 (TechCrunch, VentureBeat). Its API id on Meta's Model API is muse-wonder-1.0. PolicyBench holds no Meta key, and OpenRouter's only endpoint for meta/muse-wonder is Meta itself (provider "Meta", serving meta/muse-wonder-20261012), so the row runs through OpenRouter as Kimi, GLM, Qwen, MiniMax and Inkling do. Priced at Meta's list rate, $4 / $20 per 1M input/output tokens with $0.20 cached input and no long-context tier (Meta's launch post and the OpenRouter listing, read 2026-10-12). The serving card waits on the onboarding probes (forced tool vs auto reasoning, gauntlet, smoke). Co-Authored-By: Claude Opus 5.5 --- app/src/modelMeta.ts | 1 + policybench/config.py | 12 ++++++++++++ policybench/paper_results.py | 7 +++++++ tests/test_eval_no_tools.py | 37 ++++++++++++++++++++++++++++++++++++ tests/test_paper_results.py | 6 ++++++ 5 files changed, 63 insertions(+) diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 94d27faf..204312b6 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -169,6 +169,7 @@ export const MODEL_RELEASE_DATES: Record = { "qwen-3.7-max": "2026-05-19", "qwen3.8-max": "2026-08-03", inkling: "2026-07-15", + "muse-wonder": "2026-10-12", }; export const PROVIDER_LABELS: Record = { diff --git a/policybench/config.py b/policybench/config.py index ca471f6f..b829b466 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -107,6 +107,12 @@ "qwen-3.7-max": "openrouter/qwen/qwen3.7-max", "qwen3.8-max": "openrouter/qwen/qwen3.8-max", "inkling": "openrouter/thinkingmachines/inkling", + # Meta Muse Wonder (2026-10-12). Meta's own Model API serves it as + # muse-wonder-1.0; PolicyBench holds no Meta key, and OpenRouter's one + # endpoint for the id is Meta itself (provider "Meta", serving + # meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/ + # muse-wonder/endpoints, read 2026-10-12). + "muse-wonder": "openrouter/meta/muse-wonder", } # Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model @@ -161,6 +167,12 @@ # inkling: $1 / $4.05 per 1M input/output tokens (OpenRouter model # listing, 2026-08-03); litellm's price map does not yet cover the id. "inkling": {"input": 1.0, "output": 4.05}, + # muse-wonder: $4 / $20 per 1M input/output tokens, $0.20 cached input, + # one rate at every prompt length (Meta Model API pricing in Meta's + # launch post, dev.meta.ai/resources/blog/build-with-muse-wonder, and + # the OpenRouter listing, both read 2026-10-12); litellm's price map + # does not yet cover the id. + "muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20}, # gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google # launch pricing, 2026-07-21); litellm's price map does not yet cover it. "gemini-3.6-flash": {"input": 1.5, "output": 7.5}, diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 7375450c..026eda3d 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3093,6 +3093,13 @@ def top_failure_subtypes(self, n: int = 3) -> str: # techcrunch.com 2026-07-15; weights on Hugging Face the same day # under Apache 2.0 (Thinking Machines Lab's first from-scratch model) "inkling": "2026-07-15", + # techcrunch.com 2026-10-12 ("Meta releases its first frontier model, + # Muse Wonder") and venturebeat.com the same day; on Meta's Model API + # (muse-wonder-1.0) and OpenRouter (meta/muse-wonder, listed + # 2026-10-12) from launch. Proprietary: no weights. Muse Wonder Full, + # released alongside it, is limited to Muse Code subscriptions and has + # no API, so the board cannot run it. + "muse-wonder": "2026-10-12", # openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a # stealth preview; the 2026-09-01 table carried 08-21 from the listing's # first observed day). After the row's run Z.ai identified the preview as diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index e30980fb..1d16347e 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3087,6 +3087,43 @@ def test_local_claude_registration_works_without_the_remote_cost_map(): assert all(out["models"].values()), out["models"] +def test_muse_wonder_routes_through_openrouter_at_metas_list_price(): + """Muse Wonder runs through OpenRouter, whose one endpoint for the id is + Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens + and $0.20 cached input, at every prompt length.""" + assert MODELS["muse-wonder"] == "openrouter/meta/muse-wonder" + assert PRICE_OVERRIDES_PER_1M["muse-wonder"] == { + "input": 4.0, + "output": 20.0, + "cache_read": 0.20, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 4.0), + (0, 1_000_000, 0, 20.0), + (1_000_000, 1_000_000, 0, 24.0), + (1_000_000, 0, 1_000_000, 0.20), + (1_000_000, 0, 250_000, 3.05), + ], +) +def test_muse_wonder_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="muse-wonder", + model_id="openrouter/meta/muse-wonder", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index e8468798..801dc71e 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -73,6 +73,12 @@ def test_claude_haiku_5_5_release_date_is_recorded(): assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07" +def test_muse_wonder_release_date_is_recorded(): + """Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat, + both that day), on its Model API and OpenRouter from launch.""" + assert MODEL_RELEASE_DATES["muse-wonder"] == "2026-10-12" + + def test_app_release_dates_mirror_the_paper_registry(): """app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to update both together); every runnable model has a date, and the two From a32d1725762457960027764bbabf7ae3ee3d249e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 18:54:15 -0400 Subject: [PATCH 02/28] WIP: start the Muse Wonder release driver and rebuild release 20261010's audit seed finish_musewonder.py, freeze_musewonder.py and date_musewonder_judge_verdicts.py start from the Haiku 5.5 release's scripts. The driver now names release 20261010 as its base (5a8164a0, 47 models, 58 exclusions, policyengine-us 2.38.6), rules no exclusion, and renders new and re-opened cases on judge template v2. Release 20261010's stage, with the 232 verdicts it judged, went with its worktree on 2026-10-10. scripts/recover_20261010_audit.py rebuilds each verdict and prompt from the judge's Claude Code session transcript, and every one hashes to docs/haiku55/judge_provenance.json. The rebuilt 677-case seed's digest is docs/musewonder/seed_digest.csv. Co-Authored-By: Claude Opus 5.5 --- docs/musewonder/seed_digest.csv | 678 +++ docs/musewonder/seed_recovery.json | 1633 +++++++ docs/musewonder/seed_recovery.md | 22 + scripts/date_musewonder_judge_verdicts.py | 290 ++ scripts/finish_musewonder.py | 5010 +++++++++++++++++++++ scripts/freeze_musewonder.py | 1641 +++++++ scripts/recover_20261010_audit.py | 138 + 7 files changed, 9412 insertions(+) create mode 100644 docs/musewonder/seed_digest.csv create mode 100644 docs/musewonder/seed_recovery.json create mode 100644 docs/musewonder/seed_recovery.md create mode 100644 scripts/date_musewonder_judge_verdicts.py create mode 100644 scripts/finish_musewonder.py create mode 100644 scripts/freeze_musewonder.py create mode 100644 scripts/recover_20261010_audit.py diff --git a/docs/musewonder/seed_digest.csv b/docs/musewonder/seed_digest.csv new file mode 100644 index 00000000..2ab45c23 --- /dev/null +++ b/docs/musewonder/seed_digest.csv @@ -0,0 +1,678 @@ +case_id,prompt_sha256,verdict_sha256 +us__scenario_000__federal_income_tax_before_refundable_credits,c1943e89a6fe22c99a2682b518825fccad30e4994d0ae5721e05b06643e60b05,f1736832f26c9b0463e1b0c9401ea84592d833af117e3da1d1f2418a881b27c0 +us__scenario_000__head_medicaid_eligible,8089c5b7a0466e36ef70a540a36e0f4d2ef3e03016761d5af347bdf3413c3169,23da81291d2f22a85339bb188697a3ce2973dabd3bcdcc2c9542da0e89f44968 +us__scenario_000__payroll_tax,ad3ec2acf8cf3f51cc1b1163b7235de86faf8155ae40414008d923bd19fd4f68,2976fd7ab6bd851cfb4280f8e4bf6237fc50c8fc6bc9c0bb5fa1bef7ea5081ec +us__scenario_001__federal_income_tax_before_refundable_credits,8e481d2423d8a00d78610db517ca72ebe146b6cdda3bede219de8a605f1537ad,26332b1907accd4a10afa346bde68e6f17f61627dbcb7a66de117a3f3e78fe98 +us__scenario_001__head_medicaid_eligible,5b02a3162ff896f56d86345134b7eaedf86a92df2dc4385cfd12c4a8eea72072,01d80f20bff3223c3e6e45fc067fde2a8dacd0735cae5e387bdc5cf010128119 +us__scenario_001__payroll_tax,9823daa04787348f26f579dc51f57398cfde0e8ac182b9b7b4039777536b61ea,73dcfb1b3d84dd5872bc6df2284173bd61036e6a87f68967029a4c7c8bf798bc +us__scenario_001__snap,194a3313e5d135c9a819556944a0f5e62e7bffc7237f6028662fbd2ce2cc0593,3c28eb95c4f073cec7b6b557eb113f8e38f378e353d2767e41482c606a729613 +us__scenario_001__spouse_medicaid_eligible,0e4e93d6427a4d287d6a94d667ee53a271844c4040c62a1d59280cece0a87d53,881bc970e58f5c834264dd8cd1aef813e3b88f4288ac360c4530a2a60fced54b +us__scenario_001__spouse_medicare_eligible,0656f728338046a43a0d834c0f4142b45f0c2769fbd28ebb16e748773a99e1c5,cc4bf2ef991e1d2df6eb2dc0832bea10d4e98313a634a03c95650062870c9b1c +us__scenario_001__state_income_tax_before_refundable_credits,ec40241c45e8bfa855423975eb9895d04402eef36695e482c368d92d75b075e7,76e16fef6cedb66eaddee61f05552391deed824c858df115640cb34b9fa08e91 +us__scenario_002__federal_income_tax_before_refundable_credits,3879086195ef09bc9fbbb4224f27c4b266087792861bfe60dae6d4ef829be7f9,7b4c64608ce0152998d61f834b2f56e051899f304f1e234c6bc8d67f31d3a53c +us__scenario_002__payroll_tax,8e44f4b6852ea58a9cdbd8f3c0dfde061fec4277853883bcad021cfb08f86393,6941c2013a9ff6a340bbdb01caa5d540ebbd16131ec51ff8c5c1a4191e37e078 +us__scenario_002__spouse_medicaid_eligible,f31870852cd979c20818b8eb95109f28f97bc5884f730d8bb088eb1632540437,0614261cf2a4ae6b08880ad85cba80f2c8422e1d427e790c0af98547fabb430a +us__scenario_003__federal_income_tax_before_refundable_credits,8c65b91ee004b854a0295b8aa0e7eb2b147a7679d98ec60f97e187c116204aca,f67e7a43fee2e9f2abd7e3b8a15a1a96d5ad19f945806f1e62918acea21899c1 +us__scenario_003__head_medicare_eligible,72023d07ebc68ecacd9082cfffe74cf677f04e55c5e8ed763c05ae96a29acb3a,23246851850f9d97109d8734aeb248e05175cfd13112bd29e3601b03874c48fa +us__scenario_003__payroll_tax,7bbbfaf9883624c09846b24f41b05c655297f7909657a7b6338b59eff910c970,cc8373e601995d8be14e9a3bde16c2eb9782dd7b5b4be7b5b56ffd550e4c64f6 +us__scenario_003__state_income_tax_before_refundable_credits,887a8e3537b86044d115cf9871eee3fc94fe0d657c12db4fb74d54ef84754d4b,dadce1fd2cf0db15a840b85aec6bd3eae1f9ae0f90dde245e3fb8f352cbf8e6d +us__scenario_004__federal_income_tax_before_refundable_credits,9e983ac85d363eb014b702bb595f1d7c7244ea1e86e64cfb065f486ce116e9ea,594e83cb33e224a96cf4164e8d41c5683f2bc0c1c0e9f44d9e5bed2700b3c35d +us__scenario_004__head_medicaid_eligible,2675aa52eb07eafe4f531ab52c6d097e3d9bad454cd1576d5ff7299fd6efd176,a3bf397cbcc6f02c37862addb0c34de649b1b18ca8ab2c862ce302a9411ecbe9 +us__scenario_004__head_medicare_eligible,346ae03a8c5e9938a30641ab9f4d6567cb6c71c998f6a16872285a56885e5c8d,6a0d3eea13051465988513b7ee8aaf0b1bb0cd977dcf6fc4be3d7b538f3fa4a0 +us__scenario_004__payroll_tax,6564a474d536aec4c4493b871c44efd8d7cb1bbbb8d8b40dd8e8b20e95bc28f4,3a5985011267f36bcb9d0587e7b81a3c5849723e80eba716a88038e2fe5b02f3 +us__scenario_004__spouse_medicaid_eligible,dea74d788c0be58164bc9e89db06eafc48b2ef87a41487949db5004b53aed483,f67c8c87218256a0f55d4ff186e8c9e3e00ed87c5486c0cdcc84ba779e18f4dd +us__scenario_004__state_income_tax_before_refundable_credits,79e3c8505e4bf3b830175a7a82e1f86c3a30860d1139f78cbfdd7525c16dce73,ac22060d7debccb45bf71e79e45df9e54a606222c64945eb1abfb097b3696605 +us__scenario_004__state_refundable_credits,e65faf7c6f5dc4487aa71a232f55f12658775f0b65e6046a8721abbb20228a35,b1b6c3aea2d345d1780d1fa45fe90a705323880794e00ea121d3951b013d12d7 +us__scenario_005__federal_income_tax_before_refundable_credits,d5158b51c6d329c0c7b8821972eea319382766ec709db21f9a9cb1f9579f7eec,bb5560621a65bbe5ac1c3db9ef0ee746ee02c57f1ffe11cccea7b0bb26675d11 +us__scenario_005__payroll_tax,45a9ad0369aac5ef287f3b960ee65816107a82cf8645a1ccdf2fd62b49739c49,8405c7bead7b153e302ab89b0eca5be588ef09e3679f1b961718d708bdcf657c +us__scenario_005__state_income_tax_before_refundable_credits,23996f96494e704f4290f3866ccd8dd7b43ef41b528a296c43325a7d5c5a61f6,7653eda025bcf5be81805f01e4374537d2322c914c1c066a1dfb76445c902059 +us__scenario_007__federal_income_tax_before_refundable_credits,8830d4d963659b210068f5b4bd6b82e19a132dea099f4a457c4c61b65dbec59f,945362b7e820ec8b6f606a63583bd85f7e7683b46e94ead1e3a4cd913e54b6d4 +us__scenario_007__head_medicare_eligible,d6f3575f5ba18e234272f55c6dd9f63f19fb7eebf4e13c0647b368d8ca4518f4,3f881ff75efd3f35dfa847af2c94cb597a6fff67eaf70f139a3eddd9a9faaf24 +us__scenario_007__payroll_tax,582c724c86cdf328d26aeed0be2f7982255366d6f05bdc22aead1ff6f68a1565,e9746dc99fcb16bd82cfdc51b35bd027e9cca84a5e27da88fa8b68811c8e20c5 +us__scenario_007__state_income_tax_before_refundable_credits,bbce3b38ab1b15be53dae2e810c50bfed6ab82415efdab80c27c7b0995efdf60,ee385050ee1a01f48f66e1d6160621e1867c15de9faac56439cf0db3634459c6 +us__scenario_007__state_refundable_credits,b70bdc21cbef932867199513bf1f249c9c1b1d9bcd29c36dc8627baaacd4842a,f08a59b24d7dc4b1ae4ff60d1a26dede646b93fcf4a012f76ff27b04b47316e7 +us__scenario_008__child1_chip_eligible,a93978d5d293cfe92935b099a148d277e0c77be2ae086cb1d20efc27cf3ab30e,507b44d055f2623db2791e3a0b97195fa6353e019d482fbf7b26ff17a055d83e +us__scenario_008__child1_medicaid_eligible,546e03b9768c3db88d457ace694fee63a5d0e8d71e8fc46cf8871a619fa28362,063cbc6c13a76d76c39c448a742b7ba7f58a012d0fffa5a2feff6a5a8055bae7 +us__scenario_008__child1_wic_eligible,3de7603916921d46a583fb60f793984c2a22a885565ee7b9995d7a51c4710dab,5b9e51ff86cec1b37a9a4a1e1f2ca5d7a21b0ac1f4e5fd88ee6b06f789c8e4f2 +us__scenario_008__child2_chip_eligible,a32841e960466ea34fdc40a542b6aca12c991e6f47eed26748eb0953060a92fd,e36890a740d48f7e0f7ee115c17ecb461eaeec02547a792bec190b04885b00b1 +us__scenario_008__child2_medicaid_eligible,a9d876d93eb0a849ecf0c811c87af8ddbc6c1d819e1734eb538ba278d050e4d8,ed47b0ccb8d73d15468005c0e4119960a42faeaf74395c1f270b8b2f3d2b8906 +us__scenario_008__child2_wic_eligible,7a64b827ebe7754de190f98fcb49b3f5a8a86f4f658d603158d798fa9da84953,be6d060fec347cdf4302a2a366d0c89c94715a71a130da165e20b4a1dca9b73b +us__scenario_008__child3_chip_eligible,86f3d921a868bd221ad58746c9bf9f784b67aa30a3135a5a15a42964e4aff436,da787274666b5a020e7c15795d7beef35c29ea2eca43548681504610c455f36d +us__scenario_008__child3_medicaid_eligible,e4cdde65ea61f1e456b8a3b6ef4a8854d2f0b5ae3204f96fe002cca5c9ad3308,3c186a055604dda9bef0aec3a755bdd5eddb762baf246c3e5333430655580971 +us__scenario_008__child3_wic_eligible,91e0ae66cdd124b41e272ae84249cf23d4c6e226d1d7c068e598c551854c8dc0,6d77d109d3db2b657e9487065c82d1dc7b9ef1f30fa36c1ce019188a8e6ab6cd +us__scenario_008__child4_chip_eligible,f72543fe12a4a787dbd68130c3f0300a412f304360666158e5b69c3dcd3acb97,6848564d2585fef5fe7c09011161ffa2b111b7232b44ee054e6b0233157c5783 +us__scenario_008__child4_head_start_eligible,d05799d4149aee75cd136aabe88031f8729e9373db8d95128c207fb33302d947,4919b713b0fc80cb5c1f20bf7e75fcaf3116a0ebf57160ef7c5c6a892db8d5e2 +us__scenario_008__child4_medicaid_eligible,0d7638d57eef642f995ebc42128d99a3d526c2fb5b62de7e6c36d7b4a4078568,9707bca5e6b5e62d675fbbdbeaee6f545bc98d1d8a2bf5ad11d68dd7bdd10096 +us__scenario_008__child4_wic_eligible,81e93b029af16ea8b29b314c86ee7b821b49526c8258fe30a443ac42f71c49e2,326da5c1151f6af665504bb58aaeb5479ca5eeeca760d9a5af3aa713fa304726 +us__scenario_008__child5_chip_eligible,937edc18fe50c08bf1d28180cc7ec4f40e48de815ec078e311d3bcebace8d8aa,2fe026fd635afdf82c4519a1fad61ff70318938c1556806dfe9a605a37eede21 +us__scenario_008__child5_early_head_start_eligible,9f3b2d8d993601c7bbd21fbfefc51f64802eff6e2d6ff21f25d62cce81904de9,97a7d39317ff16aab8399cfe5826cf591941446ea1cf32959b18ce7b6983b23e +us__scenario_008__child5_head_start_eligible,575909b03776d949906e51d3c4f1d60f5b9c50ea316a7eda58ba6a04736a33bf,3132e5bed190bf98ed937baf394957bdea0d024e61ab73e51d222776495357d8 +us__scenario_008__child5_medicaid_eligible,540ef0b5902d898040af3718b670763aae644f2b1ba6bd11c9fe2ec0fc805f28,8868cb9513f817d2d8319f1222377466c24625cb8b7cfa8fba1a48586e437f12 +us__scenario_008__child5_wic_eligible,384d92fb4ddf152e224f01812c72f6040101fdc9893674450a035913387aef3b,b821cd6e9f5d1675d866f37b70025149cae771fb1b89a10572c7345f9f365e35 +us__scenario_008__child6_chip_eligible,6469dd1346fe7dcd5974bdadba4a8eb9317f6b59de0380ec7a6c58a3128914bb,f4def16caae5f96f71aad03016bf6714159193c57a8d8118793c6ca471b92e61 +us__scenario_008__child6_early_head_start_eligible,b8cf0090ac647bf2b64c32d327415b9282dd118091be384cd510a4e08127af37,81d424fff9979f86a65e7899662b85a2a281ec99e30ac89a6f0a6f5f5a945e25 +us__scenario_008__child6_head_start_eligible,c58dab1204d004755cbd5409a689fdeb19f54eace9683eeebfb351393d0e4faa,e10cdc003427fc03bc1710101cb9cb9f49ee34573816e2e308a8e3ffc45560f4 +us__scenario_008__child6_medicaid_eligible,dc0b70b2df531cb004115d50f027a45358df298499f674d74e02c5b93d70a189,214ab2db89a15a0b69f5ae2f2cf058eca9352833485400cd712cc8c02b7c04b8 +us__scenario_008__federal_refundable_credits,4e142d9c9f92b32971477c918799d6bc574faf08fcd00c728ac211f81c9dec6d,519e5dc051a016eee4a14bd9aa57b953d04f39c403b8d833b147a6f6eecaba37 +us__scenario_008__free_school_meals_eligible,62f64561aadb71fd2cfda7695d216c0625e7ab435dbdff4afd0f12d0c353c18a,92a815206251bb5249e49a462eaf21ff1e7fb3e1fbaae01a2b0841def2be9ce5 +us__scenario_008__head_medicaid_eligible,8a1564f0c291cabb2b87780aef66a5832940e45e865c7b56b5878d5de307ca33,b61da31bf48c969ac3c242f02853a1d3060f8643084090da2e3ff23c60d30013 +us__scenario_008__head_wic_eligible,92a59dacf6ec33fb005b24c9ab503994ce3de96e9a18ee76da0fcbddd80dd2cb,abbf4b1492cca1ee52c3966140730e888cd04342abbf4a450b4eb922e0487ee8 +us__scenario_008__payroll_tax,626c33c358a9c2ea5c1809fa5c7629c3625f5a7be799aa9ebc879c0bd04616b3,9ca4a0876e65a9060128f3c9408752059315ba991f182d9aeaa705619ffaf2f4 +us__scenario_008__reduced_price_school_meals_eligible,52c1c6698450f26671ddf91fa3d5e857c24126574d979402db2f7e2ad1003d14,5f7a30df6cf4fd64e856bcb5599c16260ccdc2938b2eb70aea40e24ecf26cdfd +us__scenario_008__self_employment_tax,9f4e19d631dde986e607c4d58bc589636aea04f388687cfd5cd00989ddc7a5a7,5b669021e8ce7cf60f9bc250336a693d97335cb229db0940c3b6746c2c6f1317 +us__scenario_008__snap,62f0d8ec99b7362e549c8c146c86b993112a98fc8c59aac859615341e7011bab,8ac11fab6007469a226c8eab6a10cc296b0d409c93fbb63501c9eb65bb164c27 +us__scenario_008__spouse_medicaid_eligible,cbd76a85e0b395b002e0b988ecd19d3ad38d77b61f5c0df5bd30395b67450e2b,19d57f0fee5b98fbd5dc3642ca577b3459247c8b6e59164b8f05edbbb86b1c96 +us__scenario_008__spouse_medicare_eligible,632f38d7eefb215099c3582020a75ee22ce5df9fa329beca5da6d25e466ebf9a,56784f5ffca023f279575926bf1fe365c6d6544dd1a60487622e021954c5ae58 +us__scenario_008__spouse_wic_eligible,bbf88b0edad4103bcda7ffbdce9c4859b9ba0aa763555a5567232c68ef0a8d1e,0b5455a24e0ada992eebb21a33887b7a38f5d0001c7a424fdb7a54ca9a0b70a2 +us__scenario_008__state_income_tax_before_refundable_credits,beeceb686e608a23900762839027ee423a6e2df8a9389e05dd8d0c88f8d76139,c05aafb2f3e54e2037fa3f939c06d290463b53f639c63766fbb27284971732d7 +us__scenario_008__state_refundable_credits,e5444d0a567e1f4b8d622a8bb5c94f4d5c593c11716b024e6fe23d041f11f6a7,bf20e47d5085be40b1e41050d951b428c86fe531dcb1e469a366f7e0e2dce003 +us__scenario_008__tanf,64cd17572bacb38973967f48da745366486b8d8f5bae613b2c8a9650e3ef1b72,c91c68be0bc16580d986730f2486ead82472c8e4e98ea1c38630818502f9e26b +us__scenario_009__federal_income_tax_before_refundable_credits,243c9067e20a820bbdcafac39abeabca3fa6ece4d3316044567d7878a8989573,6420da89a974cdefe8b8f652012ce92c2672e01ac659c0bf7aba223b9f96d60a +us__scenario_009__head_medicare_eligible,405b568c55ff1c1e31824d4b948ff80fb00ab44d4d02bfa7bc9285abe703f8e9,58a46ae33fd1e902d2c8ce0f93f92f0cba6bb5a0b6061f650cd369afbe2b52c0 +us__scenario_009__payroll_tax,34260f5eb3599bba24f24bf7c496901b1e00804a71b3b0aa310f2d1cd212584f,2a9b2ae1dc24cdd9af7489f87c4aa055a417e39d97cdaca05818e6afe6b39565 +us__scenario_009__spouse_medicare_eligible,25c502992c8932a7eb2112bd9d8caa000a82743a864544c26f036ff7e39dac40,76a2cffee43533b871baa2bb6353e2322c5e64a16f255e21bb79c2f226faa4d0 +us__scenario_009__state_income_tax_before_refundable_credits,d7c24b29233ba300b0590c1ee58007f1cee67d4b0a401864d3806e8d859521d4,d8be15d3cebc5ffa3f6917dbfde74c40b6be5efae67ad236ff761a9400058e37 +us__scenario_012__child1_chip_eligible,5f78a64edca0abfdef6b3607253f3c8c960614d88a5b811eba5feaef25c4ffbd,20edc876c9e6010755cff47f8d4e76c021847cef4dfc9289572ea39fc56e9a1a +us__scenario_012__child1_head_start_eligible,90dddff7843a5d2fde3f441347ff088c59c72968d1a9c24fa4fb6b1f204cdb0c,dc2210c198d414aaa124f83a59c60d021270c4f8caae3b42949deac40a240a41 +us__scenario_012__child1_medicaid_eligible,ba610b0ddf90239d0506340cfb77196e3a36e761ee9e58727543ccfa3131fca0,61ae0f73c61c3aaa837136b616be17427bf2aed3a29231f1f814771f5033f2d0 +us__scenario_012__child1_wic_eligible,e77cd5f13da5e8a407794921a9534f7ca6af6581dbe522575dc9b343313c85d7,88629e7d905733a3ff48ae1b30fba4f2cfc067f7489b052e8df5eadf80ccfe5a +us__scenario_012__federal_refundable_credits,7740d513f8593cb2f0a8e254d345e4581089d545c46d79ff127c8dd619f84571,5fc3c8de1d6d6945fc24f545af9d9001802ef65dfc5e3b43b128e42c6957022f +us__scenario_012__free_school_meals_eligible,cbbc6345ac9b2b342390f2b4ad67f5f05b6d44521cccf3fa0eb236cbdf72a995,15519c0204fc12014361755deca6b28cdb67e7877080f30fff1667df0ffbcbb6 +us__scenario_012__head_medicaid_eligible,977259a2fc27fa7804e31e8cd2a0d1f7fc5f8bdf9db3b81e92e5b80bdc6839a4,246f3b374a462a06c56a51215437e56d2488d088f1c94388960eab2c4bb934df +us__scenario_012__head_wic_eligible,bcf0075d7a75af15a600f90b50001c63f2d3b1b9998e4c7cd199b837c9051fdd,a67b5f97188d9462b41ed691b75bc934ad1ae9c9aa2d1e05895e9c8cdd8374ed +us__scenario_012__reduced_price_school_meals_eligible,57e51af61f9eedf0319eb1c6e6e5765ed9eb8c48d9e295b64e4ed1fd6f1c47be,47d52e1d7e6d98130880554c72148d759b8c81e87cc4222fab9a06e9c8e65532 +us__scenario_012__snap,32780900c4c47525cf9935c1e0af0dc2a0960992dc4f3a5cd1aa281a7c4e94d1,adbca0f364c387b0bd7a9f70f08a35df6e71b1827e914fb6f01e824700a76c22 +us__scenario_012__spouse_chip_eligible,ca2e2b03a2a77572e92a11562373838658ce4c42745d2f6b0574f6854c4195f8,cc3202be5f93f24570bab75f89f47c84f238bad90eb11ab86edfdfc40e1550d9 +us__scenario_012__spouse_medicaid_eligible,381bc83ff39fe00845ef98bbc58821e897b2916469e4f310dd86e093d6d93f69,766adafd08df7490d7469ea1ca4e8bdd712f707a5982508777c47c7a29a0e02c +us__scenario_012__spouse_wic_eligible,26c85250889e2c33332d350fb331cac26d5213d63ea806e88dbf0d4b5e30ce13,90ae25f3122f8f48fcace59e58f4d08acf154763165613ced5520f1725cb6039 +us__scenario_012__state_income_tax_before_refundable_credits,2bb19b10949c5f34f308c3296d315be57bda75ae184c5a56bc9efdcb5758327a,f946db2691209ec8a6ecfb5e5c92f763adfd9cc1b1a04651036bb7ec20eb77e0 +us__scenario_013__federal_income_tax_before_refundable_credits,49b5758bc0ba32cf35934c72284f487c90039703c9fb9c78c65e6ff6c0b2e59f,9f8c1f874fb0670067b450356b5b6040463d98ee3ccb9c6c4cb1465fb71c3bce +us__scenario_013__head_medicaid_eligible,46f55f7fd62870bc3503f9b6fe081e3be1ca4c1cf9a3d60d1d2e8c678aae8aa5,5a252dfc9466fd48248f6e8fee88e41918945798efcd6f781d7d9473e5f529ef +us__scenario_013__snap,9df47c565cfddadba7768500ce9d00aab65a603c8067724499d8e24405629668,fcbd4e34d6accb57de06df7a25432dae8c8edfd25c2c3ec52ea2aa9f18ebd26a +us__scenario_013__state_income_tax_before_refundable_credits,2b20bd6903659831c0640288c3a9df49286aebfa24220f5d3df6e5b86a73b539,df10fe12419c9927b1d23757b7fda20bb6debd2ef892504ea1aa3ffd08f0652c +us__scenario_013__state_refundable_credits,838626f1d3518944b57f2b1efa869d0bb51d0c655f5972220bad6dcd3a0b8d4e,4c6d60213e9ad24fe5f043da7bfd5e98078d355bc223488adc905503f93f8e16 +us__scenario_014__federal_income_tax_before_refundable_credits,264ec8f272456c70e038bcd5c1cf16bc0b07ead1276471ed3a2f0ea7beb1d214,b7519c60dd34d82acb99e8554d55a5f798a8b18435508bf9a8b08ff317925bca +us__scenario_014__federal_refundable_credits,77d73d79de6d24641da717e264acaa4bdefe11ad22f59daf6d9b1ac6015162f3,ad32821e5adc7fcf36c1d1587bf10514d6dbd6195cf48a5945605a22db528f64 +us__scenario_014__head_medicaid_eligible,313027647d0287cb666cd7e527d232801b09a99e676b451760ae3008bb6c28a4,5ea332fe1cf93c010ce785127c00d0eccd326b79d0e15b1d7bed20f7905e9432 +us__scenario_014__payroll_tax,69a26f3b5f4c84b500f982be577d094ebeb46eff9c18e7d54d7be54790e29f5b,d659eefae4b3e438c7134dc28d00ca2e5148e39b25a59af7dc25189fd9ca785a +us__scenario_014__state_income_tax_before_refundable_credits,80fda80baa74d25c21d45fa438f9b3a5bd0008be4cc7fff8b1af0055edcb47f0,2f3561012e32038914ddcfa2ad4d18501c361fae58b338a4669edfe9e67c9c60 +us__scenario_015__federal_income_tax_before_refundable_credits,699a87518882b4e6556f2bb230f5abe18f8fad549a257ce7612a07c6e19315f8,f4b21ebd85c8bbb6ee29eb6ee7004db26fad2dd289bdde6f6d5a2d067b1bbf7e +us__scenario_015__federal_refundable_credits,9762bfd026cc4e732c792f3b5f0715c2b19cc73284c034b388d311fc3d2b24cd,20abcd0302b072f8fb64b41f73a46de45b9ae1c0e56f131e142f11a7a78037b2 +us__scenario_015__head_chip_eligible,d7ceb52a5d6378b9178c11b36e22f119ed4622ee673b23c0e55ec64114fdf9b3,b2bfa22e6876c72ccaebc38f904d41cbee831b190c61438ab9a8fa8f2803ba43 +us__scenario_015__head_medicaid_eligible,16802b03cfffc8cd49b6414abad9bb1000f2797a76ffe6e6bb67b8c0473cd134,5a389e3f4e8f93e1caabc888cded222f52155a41df43eebdcf41f7b087a04d34 +us__scenario_015__head_medicare_eligible,5febd3eba3ef6502f1a5c9d44669b319d529e83e3979c3a1b6a053904ef43953,64f9bb5d5d53143e09d031c5050319e5011937c6679d0f4ea3acfcc482483c37 +us__scenario_015__local_income_tax,00bf68e6fdd682fa92e1eb11d4777e3b7fd1a3790bd888f61c67d2af9a9c199f,e2f324f633297e6cfcd3832de8cba9d5a71b8f022d3bb0694dd30a0cb6e9ce91 +us__scenario_015__payroll_tax,6a77bb32e0efe4f1fe6c73ab04dcc5a94066b37b2a5453a0fb604b8eb260ee26,ae846d93a2d37143ebb4bdd6ada238f55427876a921e626fccafcdb72a37cdf6 +us__scenario_015__state_income_tax_before_refundable_credits,a6e0ae7db6a34ce88caed3aaa8eb27526642c281af513cac9109f5c69e532741,16ef6e0393563311831d4593e17dc020318ca3291aec6112fa886a44bd3cae42 +us__scenario_016__federal_income_tax_before_refundable_credits,0fc24268d6629a55279229096707458ae759f7da54d6ac37dbc00e4df915bbe2,d12b9af1b9423654b1eb2f16e1378a37ca6a937e8d13c2fc970bdfa0560e6df8 +us__scenario_016__federal_refundable_credits,74c125d608580696a0127c7ed959c60eeb94c73e5124cd773124b191b308745f,753066bf7ef3c17ab8edb6d9371ac9031683926746432636369e9874feb389ee +us__scenario_016__head_medicaid_eligible,34f8bfcf0c92652d3d292df745e1155d766e01eb782a35ed6599aa7362aad3a2,b87a14633b6c862b2fb1ec6a7294353d6528aff896cd47c55746bf27d893459d +us__scenario_016__payroll_tax,a1a09a78f901c8a7ddd61c5e85d56187e7b9ef17e1ea31d093a8d7452d903c31,6de254b0730aaba0aeca8fe7055a17dd0d255c0897a658d03251fc983053d506 +us__scenario_018__federal_income_tax_before_refundable_credits,e1f69393d34e79c43ec690c3899f07bfd7692b372c962b1ee49651722f091c3b,2e31dc076b6f4cf682f88bbc9860ddcff3b37f9552d3ac4d51fea7b48d4488f3 +us__scenario_018__payroll_tax,6a3ea93b3489138dd01b6be096119b9a3f0602c388f5b3d927ca33a356e96de5,dd07f311882f306f06fcec6cd99e9358edce470f5b1f41e470f9c071b53a2047 +us__scenario_018__state_income_tax_before_refundable_credits,d5f7f7e6ab5051bfdc5df27a613778e29adfa743d628b5734bdef3488a634e84,6a2f8b3a2424f0eecb3af8a0fa6fffe2947c3102d7f7317fb1d7e14b303f8b23 +us__scenario_020__federal_income_tax_before_refundable_credits,3d56d4656a2989443d286d4482f8e9539ed381621e968492b0a7a227beb0d128,154da3be56f8792356ad68c1df2e687bdce6098354aa78c42b55df707222504a +us__scenario_020__payroll_tax,b572076e3330493639c7c9bcf127beae6c01fec3e70d0f923e451aefa3ded630,7ddc85df8e77c6d5ac46b2c1a720c057a447419d599f5b868e68c4e75cfb65b5 +us__scenario_020__state_income_tax_before_refundable_credits,6428e2ca34286ce588df84f3c842d5e27e9d214be9ecf690ddfeec7eaf97344f,dd784726d5743c8a7967fb027d6088c061f2389ce762e0ff493c967ee3aaf445 +us__scenario_021__head_medicaid_eligible,0489d1e0286f1a0810e20b940add7243f7330750ea8d85eeeddc6db301fb41ae,d340d4920b69e4f5ae8e4dbbf165e39677da72244115266110c91844236abcaf +us__scenario_021__payroll_tax,e3ae3bede8198ce8a7d1360392a0835e22bf9283e8ef0a31e6c3ce7395c868bb,7ae9f9a54cea4a2f6d92063344bd6132c8d7feae82cedd64a14875b2ca0b1518 +us__scenario_021__snap,835e04d088d82c85644fdaf4b0324ccda77fd258e69759f3ca916570d6feefd6,981799807d4c70f1c827aa4f94292b61a3cf6f1d7a4142ea19557d9557aee098 +us__scenario_021__state_income_tax_before_refundable_credits,5916cd9b6ae77aa81185ad1717dd28bb20835e4c216482dc5a05a661581e9c0d,d117348f288ef0fe5daf6fdf3e0eb4f2b620a83dff20859874171a8c3c4f4674 +us__scenario_022__federal_income_tax_before_refundable_credits,8150bedc96fc372abc3aecd3e4dad0a8e8bddb16e52dcde7212af3f56d001515,7beb1c74f6cf134e461424cb7adb52ff8d2c0248268316317bc8c1f2a3e9ea9f +us__scenario_022__head_medicaid_eligible,35a31d563430c21f8372ffa8490df7cf4d85bc4cbbad8175355aeefc62960bc4,c5beefb6218d89d0f2221698b4cdf2eb07f7237aa5204620eff8e71a43cd9e9b +us__scenario_022__payroll_tax,b6c1b2c9c0d54874e6e9aa7bc222a8bdf9afb87c009c4f531b8fe4b8599a606d,9761e01dbb3206827079781a3e369b1bbf7b33325d7792884ad7f36ae93bbebd +us__scenario_022__state_income_tax_before_refundable_credits,39d7184ce67cc967e4f96ae49fb90ab607c14e0715aab5364233b19c2bed6f8b,e53852d0b72d09bd0af25189a039828bf69b14d3bf3e25d33656995c9caa5316 +us__scenario_023__federal_income_tax_before_refundable_credits,fccb23c6f0195491583ee73635d63bcb209ae7a0e0d73272115180a3691fedb4,68703a97a7f9ecf4a0d1eb64dd7ba9c252cc9cb0e833d204f108846a70278321 +us__scenario_023__federal_refundable_credits,fd3a3004e6fc4808b6cda239581f0631439e72164c16b8e3655de63ba34c621d,8e9052d3617bfd55244b74954371185d36d30b8be607f8ce5e7a2d442af300da +us__scenario_023__free_school_meals_eligible,28e17e05d39de7b8957cd7237eadab9663714555febee49423c59fc3acd48a48,ed682d23132b5d3cab55b174133edd278e9e69116ded256002ff75b0fe508bbf +us__scenario_023__head_chip_eligible,7e5a7ae0577899811db282d1194d037bdcc6d1f3bfbc787d590df0977de38bbc,7e2ead8b1a61e568e5df9a5520e0cbfb554fe62d32db0e0548f5ebc4e0095edc +us__scenario_023__head_medicaid_eligible,190659f237d1dd022a966cf85802b937923c5be4fbc80f3d5a65cad7fd8927a7,d1263a6c3cfb4d629b05ed061dd0c7f9d1ec173c4951232fd0f7339d5b3745cf +us__scenario_023__head_medicare_eligible,327370c1f121b297f87d80f70a5c68b886989d329ac6e15e53089e57c29f56e6,fea3f5ab50a0314400d016ef46321a908942e85831e8f6ac53af10427e2202a5 +us__scenario_023__payroll_tax,38df4bbfedcbb2c96f82d22785248a55788749232e98c350c2aa975920b57eae,6fbdba12726aa2d327d1fa0cb1d1a109df45ec606d008445afd24d7eb8d4e216 +us__scenario_023__snap,66d7c9331499cff86a3e54cfb8e8a17332ba0be02bce66f5b05d9baaff9c2034,cbd9abdb5318469c30d4aa62f2cf3dce7a5dc43d13b67553c9e3e515f5aa90c1 +us__scenario_023__ssi,18155fe58781b257180794ae8d40ea3f566eda49c11ef9a9139128b21b6b0e61,c238e5f6f464eb241c29970e3ed89d9b5f0364be2889675d2365cd62e51d3c21 +us__scenario_023__state_income_tax_before_refundable_credits,fe8c98e1d62dbb5bc955c933c77ec4210a9d8975aa54733a3e9cd232552105c6,de4c2be8725adc9b4f51d4d882d77737372875c9b323f7038260d66fea69a87e +us__scenario_023__state_refundable_credits,434f2e6d5457bf852dc5cf9ce4d85af3b8b03d173b1356d9459dfa9a6b0340ea,f1a357ef4a6b1adfb2f1ee07be7019147feb51a0fbbc5c027b0262cd42297d15 +us__scenario_025__federal_income_tax_before_refundable_credits,d9e73b237259924b17207bed8b112271109939e8def3b9543ff88eb71f24bc3b,e9368fc9ac5ba87f14cd81f4810c4ffdc7b89cb10b86564b20a39cfb72ba94d6 +us__scenario_025__head_medicare_eligible,0cf47f2696153fcb224785652ba40de7ae34723671fe6f6e45582e5b80cf3aa7,9614ddd0f70cd0a6f0a6a38561f78ad12d4e07e9f99e0b8039ba2a12dd1440d9 +us__scenario_025__payroll_tax,e745166fc15851cda53e9847d63a153aada489bd6281212505fabd8d8b8b8d77,c5f274c05ca37d4762a3c18d2e5233214b4b93f8b355480b94e7ea480e14ccc3 +us__scenario_025__state_income_tax_before_refundable_credits,3c4b71b102f038ff3c9d164e0795d1f108ac0250c783458de52d06980a6d2c33,4bbe08e82d808c0817e5c8f5048e0101441f63bd6492fefceabe73cb34c02fe9 +us__scenario_026__child1_chip_eligible,61624d00a6f374c5e4ab4411533191b0be374729ccc3588f011f733ac172c20e,07c907416362ff6b3844b9f1a27e7d24e7b3234982e0d1014d4d68dfca39c406 +us__scenario_026__child1_medicaid_eligible,e73080fb927f4f8fbe169491fd055ea223b49a88a9c0ee2e3f2cb5162eb88473,94f16fbc73048676eeec5238e1b14f8be8bbe065f999319d54bffdeb5f9b05cd +us__scenario_026__child2_chip_eligible,16a8827d04f3b31e2402671cc1f1d083168910f998c11b3372186a663f999d19,a7678031d55bb7f2bf139402c1151f1afb4adce191b7191de1a4df7ba7985c9e +us__scenario_026__child2_medicaid_eligible,9c984b293482ecf03b4c7084746c9b025bd59dfe405e66484793dcc3d09c279e,3c34a95ffbea534a26058d49ce7f05cc691e12c9be489e58c5768c86da88f07d +us__scenario_026__child3_chip_eligible,4f5a3b1236a67a6675c7ab52fe4b27a847a94eb3fdaf03bd33ffad15e93420a3,145e9ab5c5be0e265479b0ccf2990184ed0731ffa4f290aa112b2b2322818896 +us__scenario_026__child3_medicaid_eligible,497087c1a97ea358dd7f44b2dd0bb8d111a12f931cad4c5c72adf2c672a80e1a,f211d327e48f01fc7ec3ea31cc2a76d39c46ba74a463428a6cc9efa7598fbcf0 +us__scenario_026__federal_income_tax_before_refundable_credits,2b36eae70a9ab4da2716dcdc3a22bb24a343bae2575978ebd672a95b695d0112,3081c5d9442dee3e8b955def7774ad41eb8ad6eca89debe383b2aed5d84d77fe +us__scenario_026__federal_refundable_credits,73300f41a742420ec319d8feb2e0ad5e541c7186a7cd1e21fe8f4c67e8428549,e428c0318924513176b6638a9ed00bb7fcfdffc564482f711d7aaf9090ceea39 +us__scenario_026__free_school_meals_eligible,7c755197149b28e687495c92cef81b8f223082d332c904c632de508f68d22e71,8cde748d4d95ccffd6c193f669506e64d922dd48f907f07629001f555505f678 +us__scenario_026__payroll_tax,40a7045cc13f30876297c0f3a86c8f476c96f55c5c4131243fe42af54ed43149,29363ab09f913fa9f50a6d4dbe134cd4e2edd8d4f5a0b0ab13f85bbbab28c2e0 +us__scenario_026__reduced_price_school_meals_eligible,c56c708f436584e21e84cdb3b1043ff74a4184f34c68b9ca0024e6bd52d30a1a,e0a3d4d035ec7a4b5126ac804367ea8ebb66ab08381156cfaf2a9487ea0f593f +us__scenario_026__state_income_tax_before_refundable_credits,987887e41b1a4987bb011ee399f447e5aaaed7cab48acde16ab5954c5518aff6,fb0f952f37680280f30fff8c21016479b68bc28806c48dd89f1bbc6f7cf7f850 +us__scenario_027__federal_income_tax_before_refundable_credits,9817e0642225ae620461b92156c7076ef9e34471e6cdbc206ea52009af0a8435,db58c0cbad389a3c99a75ddb7c79ef84d846a0b8fe204c91255b722cebebc176 +us__scenario_027__head_medicaid_eligible,2e7c67fa66042ecb5b36c3232312a1c290ede91dfb39945e73b12b56dcee1e5d,e7c94316f511169fdaf10cabc0944af4ee5f5f7458eeff17a6fbf3bd26850065 +us__scenario_027__snap,a03e0f1722da33adaad17d9171b40c6832855c5962e23c2d5d86612d99c076d6,97dabd25fc0a852d8ee88a23fb2030af6fc4e601f5805b391eecafce1f47affb +us__scenario_027__spouse_medicaid_eligible,808a92359b1dfd1875c2e1810c3cddf0b6b1e45bba51c8aaddb2eface709ae5c,4d2e892083fda18b2988dadc997ac40ac7f86f69a3a02a34e55e19da49ec1aea +us__scenario_027__spouse_medicare_eligible,2c1891cc3b9ce6592806b394125c16e3c792113360ea4f76e53f915ab0b1ae7f,e51cdbece42523623127b0d4e060891d12ac0b0f1e69e2f6b6893573492ca2ff +us__scenario_028__child1_chip_eligible,50f8a30784d924b330782356ecf6a545918adfebe7c26368fcfb69a966b2ac24,76fb1d2fbf42022473baa74923030bbed8e998ac468477fbaf502d9c797cdd32 +us__scenario_028__child1_medicaid_eligible,6d46f58fb551089e1e69eadb0554bb373a26cfb45a9d162b313d0ef9d698cf53,46f8fb273c2e5a7121faa7cc6c7400e7bde1242ec6eb4d9cbb9da3589b7d2041 +us__scenario_028__child1_wic_eligible,e2b0b738fdd154b649d1b5980dc348ff4eabfc0d5b109e9daf9d684bf87b5215,329224ec25191668408cfd9575664dea9f80ab4af46b27e880d39ef7cda515f3 +us__scenario_028__child2_chip_eligible,f7f16fe0d177efabc4a6f71014e9eb5215521d974a37ec84c3b3eb0e94fa4931,93a750461570fe30dfe86e834cd07269e6531714fcf57580ed62eec8bd74e70c +us__scenario_028__child2_medicaid_eligible,fa7bb7fd106751f713da0e507623ea8ee79a20709a013031a2f91ba1cd9015f7,99bae0ce53f2a4fb40095b51ee496bbe2d80c703eb2bdf57ce056c74f497043f +us__scenario_028__child2_wic_eligible,09a3a7f761bed7a37c567b54ee4ceb4f00feb5bcebf268cae5f4a60d076c0c5a,a9f4f2e7d7bd97e90e850739c911d83596c705a2a0ff275946980389a7d071eb +us__scenario_028__child3_chip_eligible,9680c066b3949302154b09c36b72eaa76d09cda283b79d6e4c6c48062098edb0,573d972c8d5cc1bc9e181e8a4f8456a3aa54f3e73f9e2bfc5a4afd7c25721c07 +us__scenario_028__child3_medicaid_eligible,9fa59b53f5be06fe828c3d8740721e7b0b4a6a5e85be5e420e12e2ca4d05b3f5,da114a927b181d2d5d2917db3d1c2bdfa3e34e8a97a4515ecf255a9c08a144d2 +us__scenario_028__child3_wic_eligible,420fc84607dad573fd38591ab7217a520c7081b6dcf5ac8055e921a3e464a9c3,1da539fbf580be5f1c443fcf9ec8699c3d7b707251b9632d398f4c9898ee2ae0 +us__scenario_028__federal_income_tax_before_refundable_credits,9a5de242dd6c7bf8259c07aaab1c89c062a9354776c7240844a0b83abef923bb,e636232f8ccc9b3bda2be5bc1b0f1d424247db0b239a02bfe1c407b74e980149 +us__scenario_028__federal_refundable_credits,b7250abdad7765cd103918677bad76f8e36138968b2c3d2ed87750e84af43525,703f153771007ee495217db0a2b7d9ed41ee0ba9b0d4bc1919a8b784606eea5f +us__scenario_028__payroll_tax,50e43cbb3711731d858f6f929c1b37d701de64edd47aa183e6574e16edbf8371,09c1dd3371add125f78ef9586396b5b6bcb45120cc27d3d182d1a9f291182f65 +us__scenario_028__reduced_price_school_meals_eligible,4c69cabf3d14a66d5904a0bd3ec08f4a4357569559d13943d7be050718cd2a89,f20465304f63c88ce64d3f446732b1eb1007efa991fa3333086e13a42e09038d +us__scenario_028__state_income_tax_before_refundable_credits,967b05135d442fbb84bba54cd1c30acdd5f5942525f9c8f4c78b79d3d4ab1901,4080dcb2782f207dbd5b93bc39f4aba03142ed9c22c9480c1a134941c3a1583e +us__scenario_028__state_refundable_credits,2f6c81ded6bf2b951bc08f32e1def26c47899e08706bf6c95c150633fd7ac6e0,17845b6db0826010f994abc3e4cbb015977cff448f1083d33d4ea1826840d741 +us__scenario_029__federal_income_tax_before_refundable_credits,1f08c254ddea02aee96773f0748ec7f6feadd8392023c2969b58da5300136503,d82f764c3c2ea4ac352b0005b98def82fc050ba5ed00b271dffebcfd163c2b9d +us__scenario_029__head_medicaid_eligible,425282aaef2058fd92b24d6002a29dd7ee40cf66f7fbcfa0f2449979b42fca15,13274b58aa487ed686c5539598e2733f0412f27a4b45bda02556ecafb8f3af82 +us__scenario_029__payroll_tax,7d9121c1eceb105fe8b84d4e37f679bf79588c26caf5883e0ad874346bc70104,41fbde72fbf3451e040beb627c841360f5d8162890827f2bc263f7c10a130c3c +us__scenario_029__snap,094f6578b1a61a88a638c8f79cfc0e728376a2887c0dcb1c765c7949feb89e1e,3b61c72afb3c066cd65360378d23594967b39d799bb8bbefef60fcf3a486e568 +us__scenario_029__state_income_tax_before_refundable_credits,fb9fb74ec04b344c173d0a6d3379ea4c755b98276afec70a306bc2e679d9ef4d,c09d8ffbab567b7ced18ee17d2dd5ede24168af7bd0329ecf1e4610175787d20 +us__scenario_029__state_refundable_credits,5fac6b0cd57c94d3a3685aa5222303a3c1475de07f1885173b0d6e1eb8980503,2cc0760d15f7d536a52b5a75ddcb0c5188e4188f56b185b9dc29464c88de1326 +us__scenario_030__federal_income_tax_before_refundable_credits,d1adeadd8022d2178d52801d265c4694e1db01eb310abedcd35ee5ffda8ed14e,c02b8fb860b1d77992d687b09041c0bb3b27f69a6c533a05ca952f055651a11e +us__scenario_030__federal_refundable_credits,57f61618e950470c1bf3f02f2ef7783d63f3407420223b6e400666257488adcf,8d9890d58f1002307af01c5417c7782021d840e477d0970642ffd776bc66e65c +us__scenario_030__head_chip_eligible,d16e5feb5fa75542196c3d0bb16efc8812b4c1acfca92e45a0249e2353dcb393,1b466d914250e38efdd7da8d464c353025eace78d75e58440e230b268f7bd781 +us__scenario_030__head_medicaid_eligible,bc56bfa6975337ccf8120ccf0a1c249e8f89a582c239c513d3ca298249aa7c86,1463b28767dd41d43f4891c49be6b7621eafe4d2c07e025d5343987e28a281d7 +us__scenario_030__head_wic_eligible,14cb1029cd2ce2d5aa69ff7505367972ec63e395a1f1890663a8e320bb540d81,90bef848418206359576f3201edc9b39e4ef763fa3334762ece5757d2f6e4c0b +us__scenario_030__payroll_tax,9c2735bb834a6858cd4b17440a48c9012b04cd50f1b89162a6219a8d2658b086,ad985121c6ae0ea09931a7f8a60593d0e9f1e164e787019973e63e03669bb08f +us__scenario_030__snap,3f7597cccb44f2494c1af740b867b917ed240380e7c9c7c9d2d3103141126871,8eb4fa2cba889e3d8b0516646fc53a581af94b342f06ba49e6240e099bdbc18b +us__scenario_030__tanf,fffc09d7657841ad6f1841b8c1720097020d7083b948eb9c8195413a1049f563,44812c88732db5b044421fca8bd9656d514d68381abb2eb0e2e511b6093c0591 +us__scenario_031__federal_income_tax_before_refundable_credits,819c34040c100bb9e43c8fd19dd3dbdfe8743863f39cf0306254c3710782a023,fd09daa78d449c1d07b0e97ebba123b7a30908b070311bda9523c736a272ee9e +us__scenario_031__head_medicaid_eligible,038f4a0ade9f0f1d3b7f3d693720d9338d536fd33ac9548a0b549cc12cbbd11d,50e93ea238f2cda312559468aeb3e80267e308733c838a4d31c09903cc4602e3 +us__scenario_031__snap,85c386606550649357eaa46008fd65efb08112f951ba98d3ad9e015b4595d103,e0c8e68d739dc227dacc6c3051e6a167438374f2633c05d548d4d2d852c72d3c +us__scenario_031__state_income_tax_before_refundable_credits,51b74fe0c64842577a891273b3d025e8d84e253e0678dd7f08733a4341bc976d,4d02402567c86e270bc6cbf1cef3fa5e1ea7c8ad77a76c401bd9f252994a91fd +us__scenario_032__child1_chip_eligible,5c14e66cefef3f6899746e2086bffda086d13977f6323f0f092f5204b24a3704,3b70a6b84dbc7a2a4e4ce03aef678af39664298ed35043130d8b632c58e0d927 +us__scenario_032__child1_head_start_eligible,d0693813aa4587fee63f6a5f0a86ccd7c7c9896ba67e720ac179503a7b85d3ab,0ce86babafa6a95898b1bd421b6192323efc8b875af4894401eb31eed30c2a6c +us__scenario_032__child1_medicaid_eligible,a74733aed6b43492d152961fe2904aeef9fc38af34bb7b8bec5a6b8b7c339915,ac20666aaf2c33d4dcf072ee5fe9c312b032f76158aff8620b09762c07a8cc4d +us__scenario_032__child1_medicare_eligible,c22019abadd0cb5caf269da903768c4638e5dae7ab960295d6f10390973e46e4,5d3f8a17bccb2dc1081f32f8c9fa073d861706d17f0b8452b6e91ecdc64e06e9 +us__scenario_032__child1_wic_eligible,e3483185e388f5b5bb16af71b189fa562126761ff6305cc4191c5da306ed1190,5a9f0c9a3160a6841b40baca3ff92e059c3df415ca722c52beeed68ea33a1dcd +us__scenario_032__federal_income_tax_before_refundable_credits,22ec163d64a9f3aaaa77fadcde8592cf7410b2cb8ca0b41667ac11d929d71949,5d0402c111321e64e389cf23362a2c19e26c3d5178cc77d2f9f7da2c10f0c33e +us__scenario_032__federal_refundable_credits,c692ff7cb52bd2df79dd75099a3966dbb46a12a7b3ad9684d10d6d8d708f5bbd,88cd884ba421dace2249090a7948cf1a729696a2c912f1521cf6adb222a362b5 +us__scenario_032__free_school_meals_eligible,a81379b2b87be0428f63c0d26c12665c1945a1bf0293f0d3c360d647e9bae947,da2079f33390650c9cfef0e9109d47866245c3097e955c286ffb6dcdce3c9c5a +us__scenario_032__head_medicaid_eligible,6b7f9308589ea2db980b50e3a9854e58c9613eb7a40250ddcee8b5a344a443c8,a0a7690508989a3cf0ea096f9f456cb1da61b4fac2c53bbf1bb1ea08f81ff339 +us__scenario_032__head_wic_eligible,149fb9de84050071b2b2a1d6057e66ef6b4d475c623b4128cadb26111a5ee512,01482406a486e823f19f8bea2bc62200ef8bbf3af2289f1c308fd3f6cd15e795 +us__scenario_032__payroll_tax,14a645942f03a815209b9534763fa10822c957e1b2619dbee41c26ca6abc7844,bf538655ee75fbab32127c3d2edcc64b8ddfc3be81c00e919afb5b2a1e8e01bb +us__scenario_032__reduced_price_school_meals_eligible,b690845477795d811fdeb1ca474367ed8914299e0501a412f25834da6d9af995,70dc16761420eff9defcd16604a73e761ae794da8c3153e61fff8ad4b6ce11c2 +us__scenario_032__snap,5f2ef8a4eb8fe3eee12e1bbe5530b767d51c04e0afc9f030ba962a67f3414e8c,f455c5f34030ed330632516ff5fdc41cfd09d0207c2ee92b971217caa5924590 +us__scenario_032__spouse_chip_eligible,abd4decc8dd5e0255658590c6f58de80d8158845961e2b955797ea69ff077721,531600fd17e2aed79c421c6a66e381983113078c17418dedfcae5ab2f9500e5b +us__scenario_032__spouse_medicaid_eligible,082628f87e0ffee1293496fbe2bf19a82ed6840bcafa9f0e997f761e243fb817,935ac1049c972dea3bbbcd8569074b935d45419337516f35b9d07edf88abb605 +us__scenario_032__spouse_wic_eligible,4525707f1622925908a5965a705b916bfbc11621159f00d27bb31306ea43ddcd,1fe71fdecdb36728746993cd362e22470b474fa18db8b82c01086d6a136befe0 +us__scenario_032__state_income_tax_before_refundable_credits,989e22da5d9a64d354a1e002e8c0098942b9f698221da16e1d14f30f50411339,05d6027cf5bda1af1f65d5be53f09188c4a14c74658d27b980c46af46d4a5f03 +us__scenario_032__state_refundable_credits,858e34402d4b4b473d7843308f4b2fddfacc8d97c853032cd14a4bd4570908c1,a73d2e58cde17a3428ec253f40e7c3422502a8b20a5edf4975cd97b653a399db +us__scenario_033__federal_income_tax_before_refundable_credits,d5dcf9d78866c9b78b87b9e38c76cc41476ecaa1e2479b9eedff46e2fee0ea75,6337684e8c8766368358f1f37b49ba71a0915cf10b9b3183b60bf5882fc8be0c +us__scenario_033__payroll_tax,c9aa4c2d6fcb1c429d724297793a2a5fb209b50cca76c22b9f5d7bc408cda0bd,5598d0c0d0f73374a2a854fa23c96e58fc36ab3b6c6d1133c3d5028ff15b50ef +us__scenario_033__self_employment_tax,4ef0f946d4cacf90e1ec2ce6e320106fa25505c7888647178b11772ebba4cff0,c3a6d369dc2ee778f0ddaf3347abcda4fd97b6b67bc38f6a389de2dd9459c5ed +us__scenario_036__federal_income_tax_before_refundable_credits,12e2f8b805dc8403d6f527af6b8218f4de726ede07e1becfbcde2b7d77d008ae,c635aecbf7b10f8d6acd2d83abc4b6db93d76c9bdbe55f6b8c19a3be245eac83 +us__scenario_036__head_medicaid_eligible,e39b2db67aee20fed42c4d7c3892adffae0ce958931ccfb352d161e1c12f3129,4725f5b400463224b7bb95e59f46ffa390fd1aef49fc61567b7340663d72c08c +us__scenario_036__payroll_tax,11cbc8cf6911b85ef615e214170e417f47d2eea81498bbc7d15439de6c8a2c52,1c8845157fc48a774b09a022e8f09cf93403ba5a62e87f9a6c068d3374019243 +us__scenario_036__state_income_tax_before_refundable_credits,0111a30f4716105d3be653c00a765631f7900881277474802d9a9d14d537bfc7,e115b451376863639eee6497ef7c9482fb218c22db38315d72670046f2779ff7 +us__scenario_037__federal_income_tax_before_refundable_credits,34b95bae28bcc8829cd1cc183210c49c2076006722dc47a774f83deb78ff5dfd,65172a29c6d7e5dbc7765632afa4fefe6ec5d5d3a8191c6be8d2b8f53768d9aa +us__scenario_037__federal_refundable_credits,1cdaf4d05fea8cca4227be65912a106921bf9aa19cdc3ccab6636ee1428fd089,3dcd4902d9e873ac3b22fe3cca0cad8c2a5c2c66a9c58e6c641a3c98f2f6a178 +us__scenario_037__head_medicaid_eligible,9dee63f6548a1da2aef8b2dbede00a813c136e6f217230f2e78a871716f94d89,9c93dedda1f19c53a46ba9c8c49aef99255f98165bf87cc6c4088c6f72c1427b +us__scenario_037__payroll_tax,794845071e5569220fe34637c8d32b60e72a547a755adcfab63277f96dc1ec2e,06792bb759b9e082788825aab174be85f3a4ad49d16a83a2fbd915e52ffa3bd0 +us__scenario_037__state_income_tax_before_refundable_credits,f8a5aeb255a33c077e71cc735833b1944ae1d4a7b5d95d645fb95d9294c2d208,54b9e7cc8feb488238c99d13b0bdce37be112e9dc9c5520b11caca1b8d40abdf +us__scenario_038__child1_chip_eligible,e1925f16f7e0f00d7d6ce5df76e316cd6132c22d9d6a72190cfe69688211dc89,ca207c7a8003b639fdaef9d7f5a9544995e698fa44281e84af7e2155cc22e2bc +us__scenario_038__child1_head_start_eligible,4680754501a9295699acac70c8e2ba4d19b49e5e6ef9a6037494316e144f238c,94fc0d1d9f2ff635e9828b985e2b42257b9ff1705e4492a960ad77aaa9b13672 +us__scenario_038__child1_medicaid_eligible,a0fb7cab630a01a8802977ec0ca2833b2ac81dff313c3423c23f1dbbc36abc03,e1881b7c2399a917f552c4f68bb915d42d0bd605793ecbd0e0ff15af911ca492 +us__scenario_038__child1_wic_eligible,47019b4b8f2d5906ce077cd3814afabc7eac7ebf6802fa29fa20ea701a427f3c,9dfa7ccb8ec1ed65d40ef4a0f78c3cfd4f8e9eee5c8ea345f072ebac8d0e1055 +us__scenario_038__child2_chip_eligible,19b2f317f8aae78a6082166170b20059be8a377b88d9a20c410e0383fe398b5a,b1ece33f65e3c872c1daf8d39d864fdf9d3c312f9d45b26da86449cf10aea184 +us__scenario_038__child2_head_start_eligible,bd9f2415f944741d6d17f7996c266a4b981329af0a03e80f646bdbe3f656ed61,2f20dc7270a97f493343287bdde878e4f4d79599308a62740dd76aca7ea9d48f +us__scenario_038__child2_medicaid_eligible,5e50d924df8cfacaa48b506b2647eef3cdda67affbac39115f4e218e6462adb7,bac9e1a1215239ac920601f8afe636dbf7f2f9ccd97886b2b8782b66acc80150 +us__scenario_038__child2_wic_eligible,f51f3cc2a3a957055cdff2ffc37e0134c94eebf4ce7c341afcd3b25c509e054c,18eb10faf6d3616c639deec21e8248d6d7ba5b43c8c1d67158e25f5f3f668eff +us__scenario_038__federal_income_tax_before_refundable_credits,f2898a25fc1d100cde86b30d04c5fd492953e55fcb60eb1a11342917a6e8b7d8,9c27f05cabd6370d201ecd98a538d4140ab5c592a569964b9056b9fbff30d9a0 +us__scenario_038__federal_refundable_credits,8440a3f49bb7ef5ffb70c82756e36a5f07b7e606212e8634961e49393e5eb402,efae871ea4254767e578148d8f31d42afad5749c5bec08d045c99a18f5947367 +us__scenario_038__free_school_meals_eligible,e741e90cfb79dc6be87dd576c5479651f3e328374b04e6e1504319b8d59c2871,519f655a6343dab038ddbac5deeae53ced931f7c52b56274aa002e94e9b07b0f +us__scenario_038__head_medicaid_eligible,04b06b5787ccbd8b67b38d9731e78bb56302bd7659c0332d8b8d754ac8a0d4f8,0f9133393401e180cb8ed18309472ca6075dd20f575f006a54f89b7e6aa247df +us__scenario_038__head_wic_eligible,b36b1d98223d742f75c7429ef91c7b8bdc0830d30e2b23a475b8348bf7439011,ba0f763475147eec949cac234b0075cd796ffa3de5ba9526fd735fc967b77448 +us__scenario_038__payroll_tax,41a36ad7edafe1305d2f6e400d68c37d02ef575de524585953741c19f825bd1a,964767e8eb52b0ca25e7baa56a9366b087ade120fb6e98fa1611f988a1d7a751 +us__scenario_038__reduced_price_school_meals_eligible,d7c8234536ba690ecb2bbb98c6d43fdd6d98c449e4a3b5e64e82049f5678c92b,b3339fc4f1c4ae9c12e31ae368f1c163395cc574646ccd92f5b449e68d3a577a +us__scenario_038__self_employment_tax,e0effbfa9c71fffc7b672bf908206adea97d94bb4ae8636efc1e22ab3f6d4e73,272e45fa5dd4044c8c105defea5f4521461d6af8199eb9105d8c1810d3a82bc8 +us__scenario_038__snap,827794e6727ec6439ac49d346cd4e6090ccea7c1ac15349062223f2befb553af,baf8d558384a62ff9fa5fde71a1da0a7418d972391aa7b2a8abd70f210d1ab13 +us__scenario_038__spouse_chip_eligible,6d1c5ca1572b421a39f1c5f2bb5a5b30a1dccbdd125e5324cd98aa18438eca7a,7fecd6364b45019bbb174d6bab9c8d7275648455f69e46f2c9bc471ad0700a8d +us__scenario_038__spouse_medicaid_eligible,82e888d5a679a410c60fc568c94bcd74652ea758703e4ea56debb6ea2171f67f,aa2789f5f8884ea945895b166aef41802f0cd5315a2e6b624868b3c73b2c2dc9 +us__scenario_038__spouse_wic_eligible,f77125d8b776513c4ae41e8492e5396411ed7ce0107882c107de765dfe1c71b8,d096f6ba749f5266c735fcc1f6d875dcff025e7a6af86a30ffb853ead5009f23 +us__scenario_038__state_income_tax_before_refundable_credits,49a9c2a7f26080a42cbdb384402a5f86ba4c2217499b06d0b150749c56bc736e,a1b13c01246a2e19d4595269d8c667ef1ebeab01fa7856946498835013346e80 +us__scenario_038__state_refundable_credits,09e72b96f1ba4e12a2a6b605e26ffa6662f6bcb4034f3f72ee80d6731773e2eb,9785a81a921ea902039cfd3f1facec4f0b1ebeb4b84accac56b97c95da5b7df7 +us__scenario_039__federal_income_tax_before_refundable_credits,91056adf52e6a7fbc994dd1d0f0f9ca0db9c2d9d0fcf11b05b365d93205df487,eccc9b71762cf4c9edfc4cd7c618b5dd0f544d7d4c7a8a69e9a9799b133cc860 +us__scenario_039__head_medicaid_eligible,0ba5e6c2ac7901139bb50233de7b47528377eac1a26b19a36b035ce6dad1c782,a80165745287307440e6bb2b173dc8dfedaa0c99852b2b6418a9d377cc77877a +us__scenario_039__head_medicare_eligible,ebb802b330c78600500d62e4a6d7d1ba0d56323c5dba5c52a80f076f341d9a20,a06d875a4465d0eef26e9c1cc1f8c31411c0213f3ac4cf16b0488f95314fb1dd +us__scenario_039__payroll_tax,754dcfd7934899568d3f5a36bcc16455f391f704edbef9221da71be0d6c7f3f7,076258cd4169dcad780515e462c6bf2f95945ee23c587f8937813aaa8e68e09c +us__scenario_039__state_income_tax_before_refundable_credits,2c1730aa22455590cf03526bfbed9a6a0c2a33f2cc903dd5205adc5761f6ef62,0cd1b0396319c8b5a12847afc235374257820dc3047bc1d043db973483b5c867 +us__scenario_040__federal_income_tax_before_refundable_credits,3f672d71c4ee36a64503c28943bbf8241d48698219981a65c99cfbee03b7c5aa,a5ff6471b366f730d62a442214379373964d25390ca21784e79cd82e2234fd12 +us__scenario_040__head_medicaid_eligible,36f71562790ddb5e7fe49883aeaab41245d92b1a023bda582aed544727fbe383,631f4f7cbea809810139f9de12bf5f4c3827963accc309cfaa0bbe9040332acf +us__scenario_040__payroll_tax,1fed60f4dbe4e65da843db4b0bdb738697d29ee8a839e758bf0746b12b05f140,13093a99dfbc9ff0c9edcdaca52ecd6e74ee88e7f2472e1c65a45bb1c2d46775 +us__scenario_040__snap,5475c73155e3f6a912436a1fa2e06479725006da6f96de2b7b7e9c16db95c417,4e5ceb875f7b99bbf1e5222e67793919789490d5953930572f5fd1731554e7c3 +us__scenario_040__spouse_medicaid_eligible,679d096fcf26e8174f6ceb7414d5a547aa6ab2dd2c9bc7c3612c3654b2573417,0ab3d78dc2fff266c44f24738f400dd76fd110b5c2de5be0b49465d435a1f888 +us__scenario_040__spouse_medicare_eligible,3727bf190e86ecf635349b9bc4a8aa0a40578256aee6f2e4f22365abecbae464,1e6fdb7aff3bd6bee9af83655aaea163be4142ae063fb42bec30fbd283c04cfb +us__scenario_040__state_income_tax_before_refundable_credits,3aaffb2fa7c401919c52a7b99274443023dd680128f04d71f02ac961792e5960,1fbec781f81396fc6bdc866abe0c1dad865ded5cbf82e37f647988bc421b48d7 +us__scenario_040__state_refundable_credits,4c204814eb3dcdfe0106056fe29502b5cb1e7b1b6f02fbbcc4fe45876eb91ac6,55ad15c86b01c75b543c0e855b174477801ab2922a1f1af1163a9cbab7a1c991 +us__scenario_042__federal_income_tax_before_refundable_credits,ad7c9420ce79d3676f6fae0a72c6b2cf85afcb5ee938973619011178994da199,780085ada860c36c43125ac03a11dfd43557032cdf8c505d0b4f3c705e5e9676 +us__scenario_042__payroll_tax,9f75e99227eab776a2637597c8ee73180586c94b24685c374ae42f11f2d85ed6,d830248982203453fd8aaa5af5bce90e8bc968e0f688b7eec37af83dfc211015 +us__scenario_042__state_income_tax_before_refundable_credits,6260c9b25f7b4c0cd3996ce2b0c40b7bd8dc33fcc627b51715bbdab7e2e9a9c9,68eed26f001c269c63b9a788f584a673bc9b70328a0052f3e62dc477df55e8f4 +us__scenario_042__state_refundable_credits,62d14d3fc72f3b01dfdd520e2ee8ef32a6f25d5709585a976b3112cecf45cf6c,ba18c56640dfd549e76139605da2191d1a898b4f206df529506773b004cd548c +us__scenario_043__federal_refundable_credits,f28af07658f67f40423abac96b08c60a486ea8f6ce0eae0f58760ef6fe97ddb9,d9d1527e1797ba8cd94d148932bdc59352649e17a5521f34d240db5e69a2e93c +us__scenario_043__head_medicaid_eligible,e1ccc99d8a8cdac22f6bbbdfe1cc4576fcb80ed7c1dffbfd4b1ebfbfebbe073f,7b68064124c7e670231a88f2a2bb80c773c443f4e42c9209b74fc3577f1a2788 +us__scenario_043__payroll_tax,20880542a485c03ab864cabc8c6f911edbec932509d46f3f0d3576a877ded70d,7a0dda0e9550e268300a5d61435749293eb7e02a7292b1656c7cb28b0efa7340 +us__scenario_043__snap,3a7435cd12a0c06608f916f999f18caa38b75193e97cd28f540bf5e1e33ffdd9,adc8d2a7f27c6c5fb773f5beb555d14dbad7bf7c86a8befdb37ca024ae74ade5 +us__scenario_043__ssi,3f2fb2e727eb13cb3dd6329b0cc4f1e9d1fb0e6e941a37be95f4dd26ef98015c,6eb8e40d43ed869b65016f02f060870333528d5a4a440d0ec199f405e2438e77 +us__scenario_043__state_refundable_credits,5c453100bf56dbb03a7662fb321ddfbf2ceaaf4e1b1d12b77840c5ea3731774d,e06651bfa881e21214841a6e66a01ab869c72e2edf245d51613e28ed9a236996 +us__scenario_044__federal_income_tax_before_refundable_credits,83c4fdee16e08f063240f90069b0c36b1941bc538f3197869cb0ac4010be2a32,1ad9cf7ce57e2e67729195a6061dac993c28b7da063f7c73fdb21219307a0a69 +us__scenario_044__payroll_tax,adeacc729ca9a98af1b6b4d238c66b8197c435ad37cbc852d374f7577359e6d1,519822c2d72101c61ddadaa2e362a77e396c2614a2e7cf4dfc45885b6154343a +us__scenario_044__spouse_medicare_eligible,327e24a5a80ae7a43a36ab7e0b5d45a0db4b687d96855e3423dbcc032c290c47,6c79e76ee912dc80f51a343cdc636f6637935304584e8747012a52bb5ac6d35e +us__scenario_044__state_income_tax_before_refundable_credits,4a0b0d92a574ff8b6420904f60cb9cb2b6e3c275459d94b7350837ab2c38f27d,38a30ebdd7cf229e4e9cf7ba2e7e9401ae546ee4fc18ba127c62ebe91f03796c +us__scenario_044__state_refundable_credits,1d5b7992aaf84ffa48fb15346233f2510c7b300d4feb2bd033e323d2fe331215,153e9e572bd287d680ad073d03707f9eaf8b609bdea363ae0613edaa0d1b7ae4 +us__scenario_045__federal_income_tax_before_refundable_credits,c9a335e6710514f6914b76f0a18f9ef0852dd5e60175516bdfb1141219761950,c42e06789c733ac8c469875e79ab4e01bd326c709157db02f50730aa0b9fc638 +us__scenario_045__federal_refundable_credits,13546ea920e4a4cf6b9c4968bf1c96122e78675bab5de11cb21b0c8719fc7c18,120817a62c28915719bb23dc4b3dcb6eab05ef11398fc2001b6df00182b46d65 +us__scenario_045__head_medicaid_eligible,6abf1498f4b6157c34f1a080d8bc781c053f3a2eda99a386f129864de26dfe9d,d76318a7a85f55f8ef1fe56dc4ff16276c47675acfe70fbad48cb6066a67114a +us__scenario_045__payroll_tax,721ad7334fe5f9a80a965997b81e9927953f095ca9fd0fa1ba16455e8b142e84,cf2d7c881b5bd95fe6c7ab897f9bd8eefcf6d5c19f4b51bfc1c66a11688d2b21 +us__scenario_045__snap,b2cbc9c82294274eaf513f55413f8551012d51b914779b579611bd5c8677cfcc,aaa14a511fa37c104fb0a09aa23d8168b5d3bb817264c468eda53fb17d6ef348 +us__scenario_045__state_income_tax_before_refundable_credits,fb4ddd500fa197f8d137e369dca30f866ff6c39e00c698bcf809b99b061c97bb,5930b0ab8846d8083ea8d718bf0cf016b880b762c6c205a70653611fe1b85281 +us__scenario_045__state_refundable_credits,864ef2fc785cc521ed3c07310a60b1c161f332857226f3ff6397ac4a19ce5ebd,54557967511412c9833f205ef8754fce4f3942b65f12d7b059f50ea213d273ef +us__scenario_046__child1_chip_eligible,8ff89b3d57248ea77c5a07f8663ba37dfa3a42d33108f6d723d07991b69d52db,00a659cd188d3cfcfe57724417b48d020a2c501b7db935da822a790fa45ad8b0 +us__scenario_046__child1_medicaid_eligible,6e2400d102ad8a21791349117e82e0015f27da5e5ef9db010e4e4d9791b4cb3c,5e66e6c3456e42b97dbdaee66c0b31cc008f24eb9482aef8bec2d381db24b0e9 +us__scenario_046__child2_chip_eligible,cc7a7faedfff2ff24e739d9f42a0f73729fb23a043bc4253ae3502bf455f4541,a0978af88b17f0d2d5ea14929ea6fd2122fd1cd3f4a1ef2925fee7a5f5d57994 +us__scenario_046__child2_medicaid_eligible,632e3a92212e2e4a7ab0c31ba2fb09457805de753d7f6440508da00ba6d80011,d283b472cf9db20e924b4e9c096904e1f21415439992f817528fc6b59e9c379e +us__scenario_046__federal_income_tax_before_refundable_credits,5fd995e4e5bc0cc2a5e5e97b90769f4a24f31bc1638a83a46103bf547022d49d,a197f268d2359139996d43ac45b84c416cdfae7188c4d3e1cb82901800f32404 +us__scenario_046__federal_refundable_credits,522ab6d7784b46476362327048f87ed64145f4c840d9c64616303ce9014b06c7,7b8d903134a9966d7dfdd8803256224030620b63744f2c30b4dd0f4da1e3f490 +us__scenario_046__payroll_tax,1e60d329f42a14c9c655c42f96c11ee11b7a2dc70d6f1f0fc6e34c8a7d2edef9,fff22ccf33680808bd5beda05a5caae3e8548dd0d886a282539998f7009d7932 +us__scenario_046__state_income_tax_before_refundable_credits,fb180d71e3e5468299518a204824aaae3f770c6eee51d01c85345811cd08333c,5ab22390b0953a97f285d5f732abf4af983cfc4ce4f01b899e138c720988037b +us__scenario_048__head_medicaid_eligible,af465bb558082e0762f364dc5ecb6e1078dc2d557900287625d73e148ee36773,d713f6a918118bea7d8db980d5d68170f53e3dbd891a4f4cb74c6e871e20c2e0 +us__scenario_048__head_medicare_eligible,1808e6d7862c7bf5cb34277d5ba9b52806f952d38672a674db04916198f7acca,28260c0de1c264e251b4576866a099d027eece45892cc40e0762bebf17fd4a42 +us__scenario_048__payroll_tax,99e4b79bbe41a7b315d325a46abc69ba9498be3a57979d75f54fcafed8eca589,861d1399c9402c0eec443278d01c077f1a957b64ebc580b9831ad82513c21392 +us__scenario_048__snap,1b55c8442a76c3dbbdf04129ce0247c265a73e1d7533a417fb082c4e7208378a,fa56edf7537532b4a8f2feb5b023066633921ad3d831ca39ffd362b9b3e58e8b +us__scenario_048__state_income_tax_before_refundable_credits,6b35fe6a04e8d1f85c3f85634977b1ad63ba701f23a118459656bf1ed4de99cb,1b7f8f71e11e12cfbcce38e1c085489dce9f98eda8892e85dc3188f2fa13b581 +us__scenario_049__federal_income_tax_before_refundable_credits,d51e72b2182678f4cc5d2de3f4ff0b71982f17cbd6f932f9504a2686f0989838,ea9eca35852e5dfc057fd18f4a48acda06731c4450a8d377cfb616a04366d351 +us__scenario_049__head_medicare_eligible,508a5aade6a873bc35ba9fd69f2459eb4de25e5014864cd1599e87b2131e08db,14574d1382c0d1236a4a507153300b4241958d91c0807e6ed6bea507cc5d1a82 +us__scenario_049__payroll_tax,f65338f628ea9b1f28e075fadc7f7ee7f8e3eed449268871cc1e11d4bfda9f63,449d92ac7eaa937fe90b1a72d0b9eda9ee30dde9ad2c6f79c9ff840886d14bf5 +us__scenario_049__spouse_medicare_eligible,5f2683eb144a866907725559634fc32bdef2dce3f8edc03b49e30a3442390fd8,a0b3db99aebc9aa0a46a192d8a414b15f6e134bf77efec7995267ba8e940afb7 +us__scenario_049__state_income_tax_before_refundable_credits,d0b334d42c5224a0836a87e42e8c74009fba48689e78bedbbb9de1aa532cead3,57c95a2b4ee4823d60e17f31d5c45b71c2f2ed2ca2d82f3f4de6124998f587a8 +us__scenario_051__federal_income_tax_before_refundable_credits,339e57fde1bd4e33815c49964ea8e9128eab60eeb11fe52ba340e1ca0e92b8c0,954245423973ab966a6925c305848d6d9e36059abca4d5b295d5ff4a724187e6 +us__scenario_051__federal_refundable_credits,6e6bc4e61c06b828850b4d0dfa5489c47c5486bdd1496c350a9c1ca6eb430116,ad7a510e71955355b19f8a00ccbe064d642c767b2cb5853f638c2f1108562e4c +us__scenario_051__head_medicaid_eligible,0586fb51dce2e1ed38e98e3c028de7d7bb759db9dee6c082f0d4d7a62b438b2d,4256be67fb88ced26e6fe68b169c1e1a553f3ec35e90dc5ec5e020c17cea7354 +us__scenario_051__payroll_tax,8b3810f6ea11cd7dd4221809c58cbf4b2cc3c42305eb315380a3995f285ea209,52c20d04d1d7cbf60ee335ad28822e6fe7c5e9d1fd1b7cc33af2205c6da5729b +us__scenario_051__state_income_tax_before_refundable_credits,5ddf5d321a7d37b1950d3ce2a53e509d4015f0f31e5282c94b15f1ed0f714f37,524f744c91fb09b511e995137a30465794edfd6c09e08517478057c5cff5ec50 +us__scenario_052__federal_income_tax_before_refundable_credits,bda9d830697a13251decb90893854a0b1e4a0e5e3c9528784b6ee5dd43d21d79,160f92579bdaa75197452ab92dc21c2e2c997933c55dc6ba6ac1f180f84d6da3 +us__scenario_052__head_medicare_eligible,164e17b4745fa0ef4e25039a0ef8068732ae3a546dcec246c84a61cfdcb805a2,af58ec2c96a3a746b0ad4c6eb8e1eea22489c2a579459b5bdedc9d9f0d33c51a +us__scenario_052__payroll_tax,6f9f6dfc1bf65ab3059a55e3c021bc97d612452d5ca41ba98f5edbc009fffca6,ea24754abe4dc3a73d69bef03ead75d9a392e52b0b2dd1514ecdd8cbaf7b8b81 +us__scenario_052__spouse_medicare_eligible,c12f7c907b26eac5511570a2cc0a6cd9b2f0f932296d848ee2cf68d4ace562ab,f3ddd1bcf7f2506f75121b24b4cd69805bedf69457e3c44e019602e8cdea1a3a +us__scenario_053__federal_income_tax_before_refundable_credits,0aece15329f0f8128ef89056d347e6ee8cd8bf623de96b2784cac0ad9968f356,3ec04ab9775d5495a4e0767dd4a05a6361c04cca7bf8aff2a5a31bc9ba5cde74 +us__scenario_053__federal_refundable_credits,ffede74685464280f003d47044a5f15147bd05f69c22e68ec99884f9d0f50c1f,15b94adadeaba3a4b146f4bc23a477f06389d1ff11e121d55988f5bc68c58b07 +us__scenario_053__payroll_tax,fc7bd38198ce59065233a244527aac93996100d5086cfabbc0fbe3da3be52a30,84918d4565f6b4220ad0826a53567cc64bd17f0d5a230e0076502ddcd8763a60 +us__scenario_053__state_income_tax_before_refundable_credits,c2360ffc115d1055ce64ba77c8fe3ea64bb2c6de29119f14ce957ae91377aafc,eb6b52636f2920eadab585cbb2b329dc0b8df836b94132d0dcd980ffb28b6f50 +us__scenario_053__state_refundable_credits,c3cc8ddb05d966f9c7889b87b924a9f9cdde79851f5e1914ccc8094db4b3dc7b,d192a2e31d79be9970d9a776680b3836de779248dc04ab410cf66cd6083d99ec +us__scenario_054__child1_chip_eligible,e910c6e99a7c2b82e93f415f33fa07f1b72493756e0d177a06d11fe1d0711709,99f36d276b92f49d6638f422500ed312ca85a1bd09334cbb7f6ca5c860a29725 +us__scenario_054__child1_medicaid_eligible,755ce7ce7b429ebecb898e41db45889a81e3fb532a488ae436543a438760ca64,780d9c63ac2430244647c0a3cf8b0e2e139686b2bd49b642032d3be324242914 +us__scenario_054__child1_wic_eligible,ebf420a222a27cf59b7d6378d04dd6e26f220d9d6ddcccd70d769784c844d633,59c6a581add091a9d9c395526ccf381352213cad6bbcfea5627009b0261c2a5b +us__scenario_054__child2_chip_eligible,4e56930e77963c77c11b1dcde0db4f7c0aa6b1a720b0c4e35df3c327a737549c,cd86aad7f8dd4e17624e9a99f337ab94b2e989fad77b6e326391522b8cfe87df +us__scenario_054__child2_early_head_start_eligible,1b6908a66c8df12a48e990792d1f70c867072608b6af72f925d40708412a61aa,50bb5636990a7bb880353c2079577a1ad0289d911fffbc0c92b8d978e284fa8d +us__scenario_054__child2_head_start_eligible,ac06a81aeba13974672e68ca435af710fc1c0bf6b03c7ee20bc66c926fa4a3aa,98972d0394e90d91587be10c820ec84c586539ea91a9cc646704c3be31d65e1c +us__scenario_054__child2_medicaid_eligible,7915be52b82cf6a0feb8293209271869ad1dec7e718a990c8080a1e336e9b420,ce9abe300df96a44969ae510aabd67fc18cf2bea6d1aef25e64c50655563e9d4 +us__scenario_054__child2_wic_eligible,73fa3d6dae841552259f48b922a155d40d7797027efa5aec2048a95a64e63bb0,2a20c57db837283d7745cac1c1f876366f51cbb179728799ffe66ab8ffed0caf +us__scenario_054__federal_refundable_credits,c0953192213a189f537db5dd765b70034230d7f43e1d31fc120e73eb52a748fd,48f6300e0533f8c0a1109f8b98e4a73cc78053cb8975e8bada469716e3f94f75 +us__scenario_054__free_school_meals_eligible,272b41f318402b8f8281fc7015c8f20af1afdfa47bae9f705720de7fe161304e,0fb737c11bdb437e593ac92ede33b104338a0ef2b37b50e47f07e5466823a570 +us__scenario_054__head_medicaid_eligible,17e5abc53ec3bbc718a912a326e94a39589d7e7dfaf5355521d8bf4fc07a51c6,55f985ddff08fc45b85169f79248661951ef21fe907254e639dd16ce8936adf8 +us__scenario_054__head_wic_eligible,ab128dc4837e05ab6282f10cc2b58b3c4142b6731716fdef879fafbbb264ddbb,581369a3a8cf717d35f01c271231185c5f72879d6da34d600c5ce575a1e95194 +us__scenario_054__payroll_tax,456d183aff785f6755222c483364079db7049ba9738400347935948cb4f6891f,564378153453428d1a8a02c2ba3dbfcb340f1fd1f99a201b9d6e8b2d6d26142b +us__scenario_054__reduced_price_school_meals_eligible,f12a56a4660c97ba4d7f5df29798df060b2955d9ecb829a32e6e5811d890e88f,b27fc7c3d4106e3bc6c14f3d348dad0cd19c0b7814774a1a0e0c1798bc92833d +us__scenario_054__self_employment_tax,2ebfe07a5c6540ac0761fbd0ae254e528f4ba984275bc7840d7cf5f9466f8845,57247bf738aa515db9adcd330f0be7c530a7b7ece60254beb76c1be21bb89b70 +us__scenario_054__snap,2ef3e32448ba649acc9e4748d10181333812ee026b7e9bf8eaa8c5c7953430bf,98155640e04e269c42ce6ea92c4bdeaae3430e5ff8db94ef88caff0edeb42ab5 +us__scenario_054__state_income_tax_before_refundable_credits,a3e88e18ad91a4baf3c47c93e7b6222ccecc8e62c5e2b3da93f46935d1b83b1f,3baade3fa32e68dc8facf405f96aaeb7335fdecc08574f2fac53b3de85571fe0 +us__scenario_055__federal_income_tax_before_refundable_credits,2e1e73cc4f77b3727cc5d0dd99f701195195c784335a81bff72f043740e311e6,22e9f5771d046cd0fecacc3de084c6f1a27b710035cc118a55dbc7022218cd1e +us__scenario_055__payroll_tax,d679db023a368822952b62e560f1d87d125577422fe1cd55dd09aa809cd09f46,3e3d36d2e8c525f216592da5e54c991e39f7014827b0c3e1e564fcb01207793d +us__scenario_056__federal_income_tax_before_refundable_credits,869fed4267fd38e002f36fa8c77664485d37446ce7f17227f88b651d7e6077a3,ba7abd54e65ac9d1269f5a042a2f76e33045f3746893f3f3164135cd2e0780f5 +us__scenario_056__head_medicaid_eligible,8cea184c97d14c48ab06ae60786e2e218504e4532b3d05dc5daaf77617ac7f0a,8804b0dc69774740c7590ef027a716e1d0dbda44601e402b6a7cb4ec618b8993 +us__scenario_056__payroll_tax,6531a2cf3210005ac2ae383bdcae048f99d03047c5e6de4736637171d9a3dd00,5cbddaf39f01de2b5d6a0cc5d7eae0b40221abafc0709b317e06282cf8bf6a91 +us__scenario_056__self_employment_tax,7901956ba0a795ba0c00a0c60afdc4e588328f7098b18e4cb3f85a81d327e81d,d2ed557c1f4ce1a2beead21aef264a49b77c71f1b7a5e2d8c98546a890fe7727 +us__scenario_056__snap,0841f62a55550474e31cc34cefb104a78240b982833036705488962539dbcaeb,ebff25226d847b208498e7808663d70f8fb2540aee3dee16158b53d6810e5dc2 +us__scenario_056__state_income_tax_before_refundable_credits,828b862f6768e81a9585f9ae508db46ebd09bc4c7eb4bb9dadcafd573f2647f0,2be4296e7aa80df26a2403455fb0ec1d3a9ae7fbf8c87715f4cef8fc89ab7466 +us__scenario_056__state_refundable_credits,bb2a5468d91dd9af3d64aab5096f59e216e9d23c8919ff94516d1ebce046f544,7f1d11b681e01758782d470ab3c39ce75ea753d0e1d4c31f393c61214f273426 +us__scenario_056__tanf,e29d915d8be36465f330cd19283eeb444e984e459bb631b80b28ede890585dae,c9259662a30b84fbc876aff7e14b7cdecd76b47016a909caeaf9f45390d70b5c +us__scenario_057__federal_refundable_credits,290a9cdb70ea9936a6a26ebdd4a25fcff76bc74b144f23b4a259f101436192b1,1e4fc20e30413dcf9fe24966da417aac8e9ae0bc073264e5c2db8faa7f333ce8 +us__scenario_057__head_medicaid_eligible,b24704c65dbbbd869b75808f3d75e9d21828139588965b21f6714959545e7726,69f7a36659fea087602df9eebcfda9b865b978b138e30f94f8eb55302fead365 +us__scenario_057__head_medicare_eligible,31640356d7793022ccf13f6639322e4ff8e96dd40caa05901f69fa24c3222eb3,2c9c1fafca0415c6c08052268e58cef0eedf2c7cd7d846d4fc48807102231f83 +us__scenario_057__self_employment_tax,5ce219cef2a55c537084fda62318edad4d9ceebd1b2907131db70215cf6e2687,024e7c4c5998e7c761d8a235dba4191a7ded0b45843ce21ae92e9cf14d14fa4d +us__scenario_057__snap,2fe4ce665dfcae55ac68b713f956ff8693154b30ac4be90e20f06a273c274738,fd1ddc4fdf6d6db672262811721c2125fbbb4c0e42868b2ef849ab25b9c4bfe7 +us__scenario_057__spouse_chip_eligible,40ece4779468b0368b2a671ee61fb8fcd292b92b38b0771797127becac9b3190,d24757f59ecaae276e455383add3bb6de3af7cc5d25ac40f3bba805006598ff7 +us__scenario_057__spouse_medicaid_eligible,c9d80fe640b7c9227e9c5eae6586229bb8c65c2f5ddb7bc41dbd4032fda1ea96,d1835cf06bf4d537ab87d2cf05b494e2fe3ec69cd7f16dd80bad05c1eea7a219 +us__scenario_057__spouse_medicare_eligible,9594fa0ae8fef7633cd0f0a5e8d487dec07647568ee2f8fe045c42f3db365437,3f4931cb8fc81f78b74eabba5762d66fa9e2afd0eed48c20b254ae9ed8a0ebbf +us__scenario_057__spouse_wic_eligible,cff4ad62140e6c2b165cc678d22144e5df113f38535884b399283ec3ce395179,5a5d2c00c386a9da7842d62f48676ef7c5912bbc094c7c689ac6784da04b60e3 +us__scenario_057__ssi,df01dd5a4b49a6faf2084f60412099caf41e8f4a9c0b806a1f72728538036847,608e3fb3dd7fe97c903bf813803ecb4feaf0a1660ef7b64bac2c3679d4915355 +us__scenario_057__state_income_tax_before_refundable_credits,aa152b05672b14cddbbedfd10c01edc052e23b7948665f0d2fb02b6e66c345a3,6155014a283e4a283cb844becf06bdc5e67668f64d129b48d24cb8106ff54376 +us__scenario_057__state_refundable_credits,18c1faaef19c5611435805d4940339f87a17db36fc06661dc7a473487ac4dea6,9402e362c02a61a1ebab399d78a4d54a1f0045f15e05191adc63a1d1cc58e056 +us__scenario_059__federal_income_tax_before_refundable_credits,7630823ad57b1254214e8f0ef70da1e8190b0788c40500fbd78f9cce3df57e22,bee1ada5651b68a206fee7f72a538e960513ca33b6fac5b36943fdf8506c6472 +us__scenario_059__payroll_tax,419775e1c08e04d6bf1f077f1982d6845dc0f391ee5a7309abad54ceb7d88d07,7693f0de9b470f8ceb671723561f7c1f75062244cb024eb40be97ab1601dfca9 +us__scenario_060__federal_refundable_credits,ab74a66d3dfd2ec829f5e47e906bbc0387fac68fc4c2940cd1bef51d60c079c2,f2c90191bd630870458cefc9d4cb7ed54f90af5e95152059365a924ef85b9a09 +us__scenario_060__head_medicaid_eligible,afe47b5a9a0f5298686c27bad084339c0cbaafdebb1b0103723605e8ec1c34f8,f9757bef2887b783351fb7ca4737cf76aacb2f4d638d70f60943d01c09dcec0f +us__scenario_060__payroll_tax,99b393efce61001a1b7ef18e1934ba1f186a93770060701b9932e454cef62ff3,e2900a1d724e071400b61eca7563b1635d6a2e532542fcf75e9dad0b52f51f1a +us__scenario_060__snap,72fae2f9e5738454676e15a0b6f23b9d5778ea7fb2dd254f1ffecbaa371925af,494e0586a67324fea0722cf480096c813695e55f29aa949df670e8a5d1d1b4e8 +us__scenario_062__federal_income_tax_before_refundable_credits,91cf3ed639be434bffe3e1be25380e1881d468eec906d512f27c12fff34214ec,919ad24833f5031ac9f4638a2ce5067f2740d429c683394f833c517ca2c5c4f1 +us__scenario_062__head_medicaid_eligible,174f8369cc2f5c91b7ceef7d3a98275d257d9c56be745e0a39843c7c46732b84,97a6ce548ebd03cf6c3424c54c57971602590f8731110058b7f1ceac80be1f8f +us__scenario_062__self_employment_tax,f9890892974cf662e6002cca7eedc3c2b616f76346ee872214beb93f686b071a,8f686ccfcc253f315944cfd2ce35c0f7972dea79a025ba6837f7f05defd695df +us__scenario_062__ssi,e1462635938189f55b8c9d75d8a5219a5631fb0cfaa7b49f6759e777ec44cf1f,b7b62aa2f60ca77d8300fff19b817d86faf1192700b6945bb9551d44b2e6a5ea +us__scenario_064__child1_chip_eligible,5d365c4e2795b1793b190832385354135429d96c17f03000668bc6e648d6acc1,0c8b99b0d8e1c42d4ce88430902504b12cb0b0e51d35bb6888a891ab55821bcc +us__scenario_064__child1_medicaid_eligible,a163306ffa832c6c306c97077ab18cf3edccda61d7921ff1de274ef695f64570,6ee633cf42cc5af8031cf2044791e2d91795893c5aad3536c619bcc121f5c8d9 +us__scenario_064__child1_wic_eligible,1f471bc60507ac15134edfdf89f0b723fa478de4d363a66baae1d77af86a8bd0,40c1117e868d81937384ac0ac878ad690fa6e03d9ed17d0aadfb1ef2737e7b82 +us__scenario_064__dependent1_medicaid_eligible,c085aaf95619705f66e3c49bb040b2524463cb7d4a6513d1b1328ecc183a905c,72a5dbb23c331bdf620a5b42c79dd1bb66dea050961c9e5655fb3d68aa3c6dfe +us__scenario_064__dependent1_medicare_eligible,64c4756c41aaf49f4ca5af5aa8b8ebc05fe73a1c73ed4d1ce6094f14dcdf8259,79b260b9c9748248bdfb551055ad8497b4fc7cb1d97da9bc50d31193b56e010d +us__scenario_064__dependent2_chip_eligible,bc53b95a138fa2c27903cd3b5a3507c9d4a3c97eee1c17172fa5c27c31d01d06,868fc9a16564705f61ad6a51476a5c558fd92ba01e7e3697038d1eb5dbb50575 +us__scenario_064__dependent2_medicaid_eligible,c482b5c61b5624c64f9fc25b2049ed0b99feb1a31917e2b5ee8e4ff894e80a47,e933bc244c77b2293a1138231cdfb903b45b479b17b43995681953fcd89ec44f +us__scenario_064__federal_income_tax_before_refundable_credits,f6c40913c583a53a403540c8ef1a88bf37caac15612749479af8928364f38150,0fb38f969bee9b4bf2758fc10bc89a32b680793d3ada65e895537222cd1887d8 +us__scenario_064__federal_refundable_credits,198803cbb4e457ff11805fc6fb99ed0efc12cacafacdb48b9d7d5f0a6c2372a9,25387f1dcb0642d4ac5e11ecdb8af787451c7d12734afc3630988376e4bcced4 +us__scenario_064__free_school_meals_eligible,e8e503f32843f5a4cd62293655c0eae89d7ac85538d77b9ce9b944efc647985a,c4257c9d0d823e5d54dd3988a6cf87a6bafe76ca4091c94375cea5ef418a0c10 +us__scenario_064__payroll_tax,793f11fb2e4264c06c8bfc04b6481d95a8ca78c704dad17ad8cf3f064d713ec3,95cc2f85c5b4783ede28e16a0339e1aa2814b71d6d5481d58540b59cb82c39bd +us__scenario_064__self_employment_tax,eec323b1b8ed51e273b9ef58b1a0c588494a76e0d177d944f27402970eb18515,2d0bc850240854ac857104807b4e3d8175335aba56da72ba8dad041af6d2696d +us__scenario_064__ssi,7446e2aa6190aaa1c04a025f644d524acb3e398ea5d5068a82b5d33ed3633efd,947269714de4553457a34b370a8eefec212a129a1c6404e483d9db21e9bff78a +us__scenario_064__state_income_tax_before_refundable_credits,543d9f6397bbcf2d5967b0061382e3d0484d007ad9806ce15f9f7a198b34db93,512cb0c25fe525da266ef7444c5db3e55effe5e499a861eb99c620f97620e0bb +us__scenario_064__state_refundable_credits,2d31533eb9728b969695198c8016eb43d7226cbf317011fd5caaee4922e84664,c7dc836f5ddbe8a8ec015472b276a4d8b1c9e4dc1908f8ad1f16a8a62ef66b89 +us__scenario_066__federal_refundable_credits,3488c464832e9b71f70f3b888b7d5674bb772929bbeef96b0784858f5e6e3ee7,dab99af16eee28a7b865ef93c51301b5b1576455d5e8d6530131cce5a83f82f8 +us__scenario_066__head_medicaid_eligible,6dab1279c40d43495309d9c28c93ec8f44cf7229792ce2d3b612c62453ead0a1,9e1e171ff356566c0adc18804de7165dc84df48ca50bd25db558ba5962cc4e73 +us__scenario_066__snap,ff356e52dc68566f3a0791734a15571d5787e62c5e394d123a75718c6522ff07,2344c626778ff071da05cce666d262d80cae0621b70e0915ee6cc41de04b75cb +us__scenario_066__state_refundable_credits,9724860d24732c5128185dfe03387ef81d99af4ee501264ec04091df93985b21,c4bf4bff594fbc65eb8cb19015bc9fee7b6a6e1de057c5d950bd11ef00c74f1b +us__scenario_067__dependent1_medicaid_eligible,9cbc76d33e89a9d62f135f2b95e8613f4f9d085c4bd2aba9fac6ee7f8c317dab,fb827611e784dfba133a4174709eae2f660a3dde77563313e9821e2d574097be +us__scenario_067__dependent1_medicare_eligible,7cbb2316725e09d821ad990bb3a9b05e91b8ce3be2e9c3c51edf5fb18648e325,734bb779211053f76dedeaa438166ff900e74be67e19e163850a6b3812d8880a +us__scenario_067__federal_income_tax_before_refundable_credits,8c7b1db90642ddf68f21b300752931545fe72caba1a597757133f8499619aba2,e8b41e5f42e6064c546bfc46e9abc1f0489939206bb03013d1b4bdfcf3f3c38b +us__scenario_067__federal_refundable_credits,d029f05fad8543b059707f9b3a0b2eb2673f9ffc122d33c166eb5c02d89fa187,8a04a288d5a4c5fc9367a8f6abb9bccc8f388402e23ff7177c6ac8e55bd20ee4 +us__scenario_067__head_medicare_eligible,b52a10fe439c3a0d9fea4523fa7e5d5f5eba94fef9878311a24196a2b36c480e,e420b9cabece63aa44e206a9f4148ec01522c8a821a49654efbbb8d30c62bcbe +us__scenario_067__local_income_tax,5b470a6d46dbc5ebe963e5c358aa77d19ce104c608df400f7a85f28907457406,14c23bf7f38ae94b31c2f01fda8774837e4b599dcd43f38e2d21025c3cf19531 +us__scenario_067__payroll_tax,47bf89f8663077a8d90929ab5ebe4c70b9f27a7c0a6e4055067d781e4d5fc5ff,1349613eb90210edd684be71c1cf5d8149f64fcf09eb4e1b7f23633fde1133a2 +us__scenario_067__ssi,0659da5aef8036e3e1b9b1ffe87b8b89e96bdab43ece0fdda2eab1e7adae123b,d8ead7f6d91041aca7013a1313ce3350fe4afada33ee2d5d6fcf9b08ac8e24d4 +us__scenario_067__state_income_tax_before_refundable_credits,a7ce739f90dae7c9f0d3f81592ff384e8e022d2cdd74be15b6dd86fe4befbe01,6de95aae28165e100da33a4cd7a31044b8bc3c4263cc48666d07b3fadcab5e3e +us__scenario_068__federal_income_tax_before_refundable_credits,46b0d4e8322b8ec2d2d470abb7f0089633a3abb859349dbf91c75b7ef1bd53ef,9dce4b467687663b989a4922ad1232ad280e549bfcebe168bd53de6beb369828 +us__scenario_068__federal_refundable_credits,6300187f606a96161e337ed0da3623b89e0040ed068c4898352bc2061d8d93ce,ea4b061ed07ac0f610687dbe9767a5d9e626851ab1757771493c60a7e358de4f +us__scenario_068__head_medicaid_eligible,19f5a72cdf5d984fc842871eb8607c4b549466018bb36b6f04b7d6be921e3e16,8f59d588f88ebaf855be9b388ad5aaa5e9b2b3e52b202227033bf540ebe0ecae +us__scenario_068__payroll_tax,a43ca39d6de4cb7807845586da8bd87f1a96badcfc83fc6bae9dded21622cac3,765d5220e322360d9d68a2b808cb2d098a013cb6b7db45c43123b9e007fb84bb +us__scenario_068__state_income_tax_before_refundable_credits,414748235f231fb9316129e0ae2b3d14ee2fe8f18a6e011e9537d14bbef7ceb1,d7e22cc43ffdcb09f90fe0b94f51d6892460351d2151da1c4adad04afd88f3a4 +us__scenario_068__state_refundable_credits,4a19c8a306292486a5a2ffb5ffa7d23db6eaf519425ba6ebe18a427bf7d76adf,a90a721260c18f62ea2de2c672d2320ab7aed3bba2a75e3a74e53f15e394abed +us__scenario_070__federal_income_tax_before_refundable_credits,c070792327ce4e149633446cdfa69a16c231401f9268befe9a6305e1af6f59fd,5e8dbb962ffb599aa2400add459c1d9c3a1d888c9fa5de60eeea904a8bbda31d +us__scenario_070__federal_refundable_credits,6c984702dc99967234e1448f7c37a55d07dacf5cba470a6ddb36c9020b4350ad,c15a83a6353cba54edff881cf585fc3752a941320ad36c3ac3b387cb95f0119b +us__scenario_070__head_medicaid_eligible,3cb246086d3f2633f7559ded9b7adae70eb773fcd67223e846bb08181a8f068d,42998370c8e6bc126140806ba78054929b2828c1fe2f92e7e41bdca17c300c74 +us__scenario_070__payroll_tax,d3830d6de8663afb49d463f3951e804b12685f2abeb76d570f9443622d758f59,1a49a2cc9cc655655f1cd3f580552f4939080350e7b000214a43ce7654506c3b +us__scenario_070__state_income_tax_before_refundable_credits,c2fa1e225bb2cda9f138ac3a7640e519070907cf5dad0c9782e6b943f63ec164,e21b0d38d07ba0dc368bf37f2aa6ce0cf3859df1cf9b095dcc5550b2c292b259 +us__scenario_070__state_refundable_credits,d6a8f5765042bf78f21616b0c4b87a51827ae0a769a0b39c7448507e2a35608c,678dd72b80d6ec456f5fd815d0e0ead8894860be0022e5a087d5122a45330d94 +us__scenario_071__head_medicaid_eligible,ba99ebca759d39acd30547f4bb2f06d2161374720399b22be919a2983c7a928a,dc1212f95319023a03f94eae3467b623a7495aaf08bdd995ac6ec7d359391cea +us__scenario_071__payroll_tax,123f3d610b3d0ae82b3f4df3a9f5bfb960671a64c47f6bbe6396c7f3202adec5,6616462501479e935116425369020023f1a3e8c39d4d6fd3f53cf00064ff2551 +us__scenario_071__state_refundable_credits,6840e2782fdd59bd46ce7d5aac8b2bcbf2d550da8bc1e91078ea2b4a85d65b60,9ba83cfa83c3e12acc3018f44d1d6ccec02368a9981a68b3afb7b3d1d9dc4b54 +us__scenario_072__head_medicare_eligible,8303ac810b4482adb25fbe1fdfbe1d0154f91d4d3cd660af81dc728b5a352818,619695951d21cfb3b3ce679f6d4057976f9277eb3ceff6fb811f83b7df8b8929 +us__scenario_072__payroll_tax,42e1e96ebaa613af49f2faf9686b75e3164179d2571f3aa394bb03455c8384a7,c61b259094aa5a9a9380f1a0bcd27f06227930baf8f1ad4f19a9ff74387248f1 +us__scenario_072__snap,99d0d803c8e9c2a40c60c14a032da91da14afda5941c513890221b41135c3847,28cab303bf314c742a14f4dcaa304f582f5d2d8cb88e4f5e34be97ec0cd35d48 +us__scenario_072__spouse_medicare_eligible,00c27b71fe5cb873698511e6cf4d986e5e6b3e25d067b5b633628f441adc6cd1,31386bfac7c095691dbcb3ec771403a326a679b4ff0908eb0803760e4e85c31e +us__scenario_072__state_income_tax_before_refundable_credits,4eb78404c03e372ba9c32dbd8c41b4e306a1d07a65050e5bba6f56fd6b1823ae,0c694480a2a62dfd8aea7f709e1feaa39a5593bcf261b9d316f85f66effbcf4c +us__scenario_073__head_medicaid_eligible,0ad62652d3553a704ea065d65b8fc242d0a255be297af0d390c6c87beb56fd64,48965a2fe7636d7972f4aa0061cefc5d3304f2c4e667e4d1155636292ef8a70b +us__scenario_073__head_medicare_eligible,0a1fb04f38e096a055cb7c73ba1455602ba851cd6c3c7db370118253f8167cd5,9e871421c786a853c005cbc79fe0284def24692131427b59955eea5516a6b8c4 +us__scenario_073__payroll_tax,62a8774e28a1d2403c47602a40f50751c068f3063a0df96772719ad687c9b7b2,dfd6807014296d6db200eef2d57f1e2a6ec3e63eb936c6b28f8e1d6e649793a5 +us__scenario_073__snap,b4e5052fbb45e2022475ac12e736734229b00c784c57ebd2130decb8e9f69b9f,56073985c932e4c369df4e016b5afb8626c1c6415f57255a055e8efee73d28f5 +us__scenario_074__head_medicaid_eligible,84befb4c8867cc4b47c5aa511c00b1e3cf709262c4ccc6c93b66d26d0f8371c4,f358889317dbe6c6c2d77f868aa2033578f7592fefcc40cb64537bcdb567394b +us__scenario_074__head_medicare_eligible,f0a99534476b8780cd94916f15dda490a55fd7c013917070cf3295d8b3e01d4a,e21095233ff1c9c56ba96883ef30dc220c12b6a9082ab80b762fefff10308762 +us__scenario_074__payroll_tax,470de108439fb9d94ca606a71cfbfa8ebe2e807907f172934333e6c93cd4ecdd,3ee8e6be5541322047c977c78a792ac03938004e431df721d89d37526fdb2441 +us__scenario_075__federal_income_tax_before_refundable_credits,9c7bc63ca553f35e6623ed395e2b082752cabd103526192b681e44b5f0fd2a88,62c7d4602bb2d32f6f3156c45915eeaf2da6be2f658a1772f2d0496f70e3cff0 +us__scenario_075__head_medicaid_eligible,3301d5f151624e892a7d200692c2f9b3286e339f68a0f52b4a14ac573d770521,9285a211798d2f119a35c8179164fb0db08cafe2d72ca3f7d195eaca3f367ed0 +us__scenario_075__head_medicare_eligible,b772cbd89b76a4e8145082b542fcb3869161d4df299801bc9782283c9459fbd2,d091d4ced680f4c21d01f489eb8fea087e509897a6da4f3018ec802c457620ec +us__scenario_075__payroll_tax,7d1c1f7e262b9e6a0c67063f196ea4d86b385e66f4164bebd70d580ea769c936,1429898520996c496b6ddf693ce517f323fccaa59dfaac79c35275a41ed74a9e +us__scenario_076__child2_wic_eligible,dcb324e790cbf06a106c3cdb595c777872c2eca48a3a4ba1f57a0b5348db4ca0,d47eed169beac46c7c0f8ad0537e54e5233515739c6aa11849ec639935e0f93b +us__scenario_076__federal_income_tax_before_refundable_credits,dc3b89eae9ffc45291dbad75cc6a7a538623cbec700edadf9a1c8e23a6ba919f,e83e13c40d5eaac576fd58938670de1bc1b9bb20eee203858de6a3ab219727a6 +us__scenario_076__federal_refundable_credits,4125a480c2c2b3f05ef21d1b4b30065f70997a7b9419fc144dbc332500f21fa1,1057142da7c9a28cef711e0e5e206c0188da6869e7c415f99eb7868c0c0958d9 +us__scenario_076__payroll_tax,cf731d2ccb49b1614772ace7fc7a47afcb1fbbd9df341c5e0f06cf1ef31d47e9,2a309f139ac532c2147fe024d1485aeb3f3181f2253a489e275bf4ca086018f2 +us__scenario_076__self_employment_tax,2d150397ae7fe5ef0a663ea7529317a421fab3c9d43adfece5b2f9dcb71a9fb4,56cb0733e2e10bebf17bcc2e51ca2ab7dd0b5aee26eecc95fdf5f74a0d625228 +us__scenario_076__state_income_tax_before_refundable_credits,d37ad790c66a0ba9491f8f1e8a5bc247b854d6077cea89abf6ead32bdff41f22,8e4680ae3a223014470cb19b74e864522c20ae9e4a3217b126068bb5a868960e +us__scenario_076__state_refundable_credits,bcaf490a0941ac20f02f335d9b5d4f80176826dbf5cd68726a0b84ddbf296c3c,b6f89c06b7b067b71609f65badb0f5362a037d32219c89ce0f18cef73fa79534 +us__scenario_077__federal_income_tax_before_refundable_credits,cbc6469c3f879b5059834f1fdce059167be41e86e69ba58a79345011238e7c65,19bcb97950e18f6ce00439f3fb7f9ca0530fc7fea0ad55e506c9edd9580f4fa2 +us__scenario_077__federal_refundable_credits,b8f45460d30c69ec4f3b5c14ff1acdc762b6d82fe9f553621d345bb5c5859153,be84ef35a55e2793d5e8a790ce803caa6e270c26003936463169e76f0c1a29f3 +us__scenario_077__head_medicaid_eligible,d72d46c9a98b4b56dc463aaa6553382adc2442bb3a184d3d91222a38c5749899,7daef34f5bd68e1748ad3a0ed17772bce6c440b4ea37f93ba8897de2d10d7e8b +us__scenario_077__payroll_tax,338a29146830eee07f27635e2e63eb6c6a9785b752b595b4cf6d402cb616868f,4d836d7a89aa1405b0da7a4b7e03057d1d94c70f8189e1727b1c84d5a5975967 +us__scenario_077__snap,42cd315c4574a478c9f5ad080d0c1020c3900f0706793796c24be4bd13691110,a9c625950a6df86d4d71c926ed932aa92f4a86f3d272012e9242f57ce95998d5 +us__scenario_077__state_income_tax_before_refundable_credits,dc10c82a8bd5cc2bba4ac13701fd6cf52aa8f4ccbed4f72d7ea6b67e71c5efbf,ac25f9a957e6df1a804ffbc0826fb5f27581670aedfe4512f1b136e2654f5e6f +us__scenario_077__state_refundable_credits,016fd89e08a42ec4a55ad0006c9bc9999c5bd0038ba6f84a328a9b40a47db34a,eb290063dd4670a7b7879f0dd59d665a52599fc1669426a7c830194bea20558f +us__scenario_078__federal_income_tax_before_refundable_credits,d0462157251cae830add57246b1b10cd7ade4bfce2398779cf587a7aafc3a79e,b2d91b793cbe19150b176bbeb6204a4b7809879f36bdfc1042a02906f84bff97 +us__scenario_078__local_income_tax,ae75e1f800c367a941799f18bfa5df88046bad46d93395e67dabb75c8020f187,6ec4769f167ee3cfcfda8e2b42a07aad5103527f4940990caf8849b3cebd50d6 +us__scenario_078__payroll_tax,bbff94fca330f1b1a09f6078971c8e7827ecc23960ea0bc7a876e8a9d165f7d5,2c3b31657284cf46b52990d81c1ee7cf26f4c535ad1f714e80210a7952bb23c5 +us__scenario_078__state_income_tax_before_refundable_credits,bba3aa10ce12471805a4c87dda26a41b9bf337a5565ca2ff88c4e660d7519cb3,2fb4cbdb4f167b71c15c97d78f0408a783aa69dff95c8f8dd78b170b850d6dcb +us__scenario_079__head_medicaid_eligible,54607808aec56f1a6d052748f5f44a5f63ebc6ba602de245a30258d5f9375819,d13733174e7a5e03ca4af5124d5a084a74817c97abf9aecb24fcd9c83b2da0a9 +us__scenario_079__head_medicare_eligible,eb612da71ac1d4c6e92ad2e300b37508511f890db7fce9d273fd60bb5039694b,48dea412783c086b1c96d15783f2da65ebcbc3d3f7d70df722d829b8ab7e7201 +us__scenario_079__snap,358b7d066755e939451e115bd684051e1dee80621ce4855a44e8070dde7c2246,5d720888ea025cbb7a906771e21385bd1dae1bc787b4fc1ef7ea92b2a336ad43 +us__scenario_079__spouse_medicaid_eligible,c380546d28efeda3bdc439bdf5eb9697e0f1f6329ddd31e18a908af6af7e862e,36397b770011d07056985f6ca8492d56f84bdcbbf024c37a9fd78151576edd92 +us__scenario_079__spouse_medicare_eligible,c57f8a3099906ba8ac283017b9a470c2c522d126a621c500daf75936be7d4c07,2e385f74d441774fba06673f0649355e114dca399983ec8c6ffb4533a438702d +us__scenario_079__ssi,933e09f2a7ad4721d29414b4472d27d7e77e7d0f3c80936477f5d2c93156d043,13431be4f0c9bc6fef0ce3f18db21bcc0fda87aaf5e500f44f26573fab6930e3 +us__scenario_079__state_refundable_credits,af85643d6d3badc459565b10d78abd1bf9dc3c31b61196c9dcf6710924995945,ccdc6b959d15ad34e6781e8d45b623ec61eaee88f9194bf000b6e4be3f6f9739 +us__scenario_080__head_chip_eligible,b9e243b6e9c037068d659a827d4efaf40abd5d121f00858acc06f269b5ae1f5f,2bebc812bdcd3afb3421d9cfcd80f85c031a54519c059a9ea5b62a5bd832d008 +us__scenario_080__head_medicaid_eligible,8db9ea4765f350366b1609d0ec8a322bc46e106eaa2ea8df277872bdf357bb09,89e224563d973dcea92e8a030b2ae805db18454dc8d918115005a5e3af0449b6 +us__scenario_080__head_medicare_eligible,492bfbaefcfc3cf4d7f171d47776f7c047bc20c284c26e9d66b6300db4a7ab44,42129dfd841242f5cc66cf5d734ce1a8b079879a1bf9d95136305c06f3308fc3 +us__scenario_080__snap,6f1f87267f9c7160cd0cf2e19417e42be61db743242b5f3ac6b4457378fc09db,00461d72bec0c63e44f74e1119a5a051bd8db0d509dddc95aded9136c2c92cf1 +us__scenario_080__ssi,43ec21461fff18629819bf6759eb8e9c588bd39df5f8995dfce3ebcdc481c8f2,84d00efc18d7c7b1daa09fe96695cb46edb33843d7af0e65f0d3d7d97b80940f +us__scenario_080__state_income_tax_before_refundable_credits,26ea6c8424e9afaac05bf877993a8cf76a402aaf6992178465395a8abf77d26b,65e8795b8cbd17ed44252f37465d7cdb50c6ecf219a48c336691cdb1fd8c706c +us__scenario_080__state_refundable_credits,95beccf4253f75c7438dbf00dfc205c887dc689e56002fee1d0b4c18105a4972,ff2fb3e2da9cb07e82d437f6d9a4ca29ddd983170e1876ad3a7ebc3d68e86b9d +us__scenario_081__federal_income_tax_before_refundable_credits,01f58ab354674e5edeacc02fda7ab4c7bd8992218e5223001c288b8a1f24791d,11819fede6b291f15a7feb97469c3975b1ebd32c53cf3d11a697df64deae28c7 +us__scenario_081__payroll_tax,46c6f5fadadab8ee0a97416418cbce9def84fcfb84513efda1a389a2b9062492,ef2fc03439ff51bc446cee92fabe93e28ebd5e988d31d2173341a7ec0fb8c1d4 +us__scenario_081__state_income_tax_before_refundable_credits,6a2d1758f67f045d081aee8b67c8f223fc6b95b94fa10ad6fa3b48cb308d6857,434ef3ffad3c4c9e92e1d1a282fd3a08018f1d7f5cf2987227bb60b86a733995 +us__scenario_082__child1_chip_eligible,80b33e55fbf3e28c423c7aaba899db4f361fd63fe4174366cf3a0af84b2559b5,731f2b677eaf50a70fe8968a3ac828a4fa48185dd863e74bb6c5a87316f0b99b +us__scenario_082__child1_early_head_start_eligible,d241170df6fc5ec2029d90f9fc36e3c0efe8f52334fac0783f87e6d89f63f7f9,6f50e20abc79a7c20c8e238114f31f4ad62d176cb1dbad5bf67ab443dba826e3 +us__scenario_082__child1_medicaid_eligible,d4c90f759a1e2b96dc2246cc21623d0d55b418986ae37fbafb34d2f65d2ef1f5,f2f7943ec6b3367fe7205f7f9eefd68110bc23da84128c82960ec570b09de8ce +us__scenario_082__child1_wic_eligible,3be2990225770847807aa5a221c2e4b8aae867b804e2e3eb93b39d925badd0f0,165c8776674e8d5807c4aa4537e26810fef06866a5ad4578193e9103628271a2 +us__scenario_082__federal_income_tax_before_refundable_credits,e2cdaab17ab3730ec11c42b32e066e2f21c5b0a9bb0351cf7eb7bde3e3bc2fc9,2b3aa05db39d4f909d50328dbccc8de00a5d42e4ffab09fb2ef9f31b2608ce72 +us__scenario_082__federal_refundable_credits,c14b0ec60660e88c83f681ac79565b9e931def8d84dcea0590a0c99e47740a03,da5db67eeced7d31f30ad4a4775297e571fac1755b5118b03629a4c3b4d143c7 +us__scenario_082__local_income_tax,7e5e56f143679d05ce5e6a85312929eb7775619d74092b959eea6c0c5ce60562,12dbd03b078e5536dfd64d9a2361d3da5bd6103062e9fb62b9464eee5a48ae0b +us__scenario_082__payroll_tax,748242baa478d1d1c9209b9c4e91d29089b5f17b93f7fb49a25ca783d8bb5100,75e0dc06d14f919cdee0a2e82a8611ea2babb73c2d62d4dbe1d507ccf9e5edf7 +us__scenario_082__state_income_tax_before_refundable_credits,b35e2afeac00102faf9d716554e4c800a8ee4dc2e3b7d0c6feb2d705b51ad7b2,248bde3027955427bc3b54651f88b86d447a2f748731ff0a17ca1e3a70479bca +us__scenario_082__state_refundable_credits,106165a155117af10eacca2a00b893d8f4e7a165afc92e7445560f2aa81b0bac,48ae58d8e85ab2ea0ebabdfa1bab08f5d8e7cf4ffb2a8fb0dbfc8b42bed2e14d +us__scenario_083__federal_income_tax_before_refundable_credits,ebfe45d55e2ac1aa9edd1ce1c1311b2972a11be334240e20605b367d4e5091b6,73e0da8cdd71ae02011f35c379191fd015f3652f82048893e7afa40e3a00a078 +us__scenario_083__federal_refundable_credits,608f65e5e547b0b5ec853ea1d43d57eaa0d895ed98f84d8ea6cf9b72a1e887f6,117ed52ffaad02a72621d18e1c940a874549d1cab93f06eb676a74db9adc648e +us__scenario_083__head_medicaid_eligible,8358e5331be3f07cb279f791570254a115496649a01a60a36c1c6df4f46be714,1e04ead6d9649b46349582885ceb3c20cd5f4592c4fa4a238d9919a7cbf2f64d +us__scenario_083__head_wic_eligible,4435607c931ecb6c4360844bc887a7bc38e9dc9afa7f19a74a94457d3914352e,9c87962fd37c542e9a6272d0dae6388a94115e560ba1bb116bd347e05a97844f +us__scenario_083__payroll_tax,c4391157a1e537f41f75ed7fd2cda7074fe1572909fcd9100b287dbc18f0eca2,a1ca26d419510251a0ce99bfd8af3bd41c00f754a7528742228d27c0b4bb2eaf +us__scenario_083__self_employment_tax,7135171a2c6cac98702a69c4b2942f96153cd6c2123cd81e8e6bf3f958f3a085,7dc7d9549fa10e089409843468037006efc72d517049285383f48a1bbbcb74ec +us__scenario_083__snap,847f77b72b10efe0c5b2f6273e5130dbd0e727511e215cdc07419ab55319acf6,cb1223419824e574d2938c3247d031169ff0e1902a6483b496d0975ddca9a23f +us__scenario_084__head_medicaid_eligible,0e5e90dc483e8be74b2d68446065a96b9c585838afcec38845f7c8bb57d65534,88e6a367b44a1e25fc365bec18aae59c25f3b077635fdb2212ff131be3f4b284 +us__scenario_084__head_medicare_eligible,8197979f49d7305ad0ff467b6d11d27ccd828c3d2d098bf90d0e376a4eae9e83,8a3f550550750e55f2faf3c701a271d21512800edff4229255fd002cb710362c +us__scenario_084__snap,31df3f430c2ff4676f627b356e2ffa8ea2a50cdd37d5f593e9daf1fcc401545a,cbe27bd9c1a4f5aea880cf52058afdf402cac8536dcdfb776d6c752ea881e73c +us__scenario_085__federal_income_tax_before_refundable_credits,a273818e33f8e7f7f84955fe2cc32f5326d7c8967ae0818d79f5486b96eaa222,ec50a0771a8e881f257d2eefcf709894a42ce36c7804ab120237b35e3dc7f7c8 +us__scenario_085__payroll_tax,d3d31aa47604969bc9eb8a80b68fd9482a43146ff55ecfc0c187b1f33f582406,10b9eaad094fe3eb3b1402f485db0de6c5fee485ca11738722c4ad76decd18f5 +us__scenario_085__state_income_tax_before_refundable_credits,2e0f74a999fbf5fb8f13cf8a7690164e2a41f0be7ed3650993edd8e3d0aaff22,c3df674a0af1d45e867c59650e0d7d8182206f1b316eabff57df900cdd1655e7 +us__scenario_086__child1_chip_eligible,132b2353232480d388f6b4fe9b18719217ce2ee58fca95c1bab94eb72f38081a,8b42d7488bda25472edc168c6cc71eb0ee1b7dd3616892e5033eaed742a301df +us__scenario_086__child1_medicaid_eligible,b93c423803d89654ef4205ecbb9450f810b164b7f0e2a494159edc8b3f4f25a5,852e7130b43982e057e00b4f4a0a49d178faff72c066725543f7e6ba5ceb86e7 +us__scenario_086__federal_income_tax_before_refundable_credits,9272ab85afceb414fc2f8c418874532f6d91b36a6570bb5231ed3e3dd7f3b993,225f0748e81e9e4434bea454fdb99a5b496a67fd9504f477508623759ea52cac +us__scenario_086__federal_refundable_credits,278696e8621e02462272b88b837c08e4a6dd4ec2518ad57360573d0fd9435d1e,101ad0a457764903753416f9aefa0295e8fb6de8a583407675a3e7e20f504de3 +us__scenario_086__payroll_tax,daa030ce5a2659241a349cc5010205281928f2367e846c278432ab7a30c1a2ff,5a4c6d0128dd3f73b6f231f93181ad95820781102c12c9e76da2ab171dce6b90 +us__scenario_086__state_income_tax_before_refundable_credits,5f8f2ae329eaae2811d349cb832db188a7c721ce9a507f2b9d213194b9b9d82b,bb0c19cbd2b1e7d51dc3eba71e3958d7cfa527bd3bb652811738d50c1c522ed5 +us__scenario_086__state_refundable_credits,ea1145fdbb6d74270c3ae2ea376d9aa3a345c1c12c125bdf8b139a6e06b0da0c,40da83d0513aa388507d9c5a60bfaac4e222f1fc2b8d970984683217b43dcc7f +us__scenario_088__federal_income_tax_before_refundable_credits,e1bdf03a8438ad2b82d2d5dffdb81acec8c63e18ea0927ad3710a23ca68bb1d8,80265259c57739dbba4b287e55eb8edc58985af7d438b4dfec98641d640d9039 +us__scenario_088__federal_refundable_credits,1b4b87155061a5567d6f438eb8c9507034b9ba0eca8b8e7386a2abb45ac5f8e6,548a6ac1712f13b8de12e2f882aef60f22c960556f3106fd4da39821f47ab74f +us__scenario_088__head_medicaid_eligible,f262f7ff9be7bddb5a00cdee6dbe87c0704c5956d0b93b5b6cd97b8400109992,c94f5e5cac556c5a4826900a6b3793a7939a53ce6fe38819f5b9d8deefc45bb2 +us__scenario_088__head_medicare_eligible,21ae3c74efb67aeeb20856ab39ece2e8545fd6802a9a4db7d8cf2e37f5da9099,7d444b630e33f0cca14a5d33e062b05801ccf72f83ae3b8c015299b54e311d77 +us__scenario_088__payroll_tax,ee728e53172e6c8d1a81d8fe48b4d77018ff88f0d75ebee0921134a06b3276ee,c1636310cee9bbeadd30d5d11afa023f948a25a9330c35f9d86b4cd470e41f5c +us__scenario_088__snap,b513ef27aa643ae954feb25faecaba19236c48352e713e0a023956d1115c0f8c,c2749c18b4c65d93ba5e259c14d9d1ebb9f1d32fa02606585f5407983de1c37c +us__scenario_089__federal_income_tax_before_refundable_credits,97bc7070d9808eebcab4580f0b8d2320fce32585479156eb233be13a590127c1,74144742eaf21fb93146958ad63f28eac70c7334dfef1fcb41a826d8d349c794 +us__scenario_089__federal_refundable_credits,76b8ffbf111908e61b6585203ae5d2b6ad3a88576729bb525d54a1dc3da1e0f8,abffbc6622c3590de80b2b669cc6b0cd2bb82bc6b5ebfcf9d4db194102637e4c +us__scenario_089__payroll_tax,3f665497df67834b1bf33fe1ad8ba3bd0b8d2d2ced08e014dcc7d4296efc4359,f891b02dc2c95d4981e136811d6076c9c4a0a919ae3288364dc32a39635c9679 +us__scenario_089__self_employment_tax,31b4068a5c1b9409fc907ce06fdba962d424a2b20fb8f3c88bb21d7053b5cc3b,db7bf9ff52dde77a27ef2490a502d241c983fb3f8662535af91719a181ccaa38 +us__scenario_089__state_income_tax_before_refundable_credits,966d4e9be835310eac25f9dc755cbe44e9329f5e6be6d2adcdf3ac8609ddfcfb,819d83f792344de3bf9d870cd7d738d9129575919ba75ac679bd2a870f8e81c7 +us__scenario_090__federal_refundable_credits,883b9d8dbff5f97cf405381d19b5aa6857773f235705f7ed3b6e633181928a9d,0b986c5eb2a684246d25fa0f7b066d3f6664a4b46d150c4cce4899a4a13c3705 +us__scenario_090__head_medicaid_eligible,975518354410f94fda216542642828d9172e924dc5dd5312a6292e316ce5cc95,b2bf8589c249f839f0d978d9f85d4d37bf37455c466db33bcf5547446b7d52c1 +us__scenario_090__head_wic_eligible,de7963217884179690525723fd5e09cabefd68b1fed7287ade0df00331ed0912,cf121c9a8d4a8cad7f923bcb84e57a761878545910ab9745e14f92cdc6f9a9e5 +us__scenario_090__payroll_tax,0241e654bff01cfc8123680d878f92de0b5c8045919cc6c7d34d7972598b7a28,ec28e561136033f45a11d8f085a74170568bb6c6bd2a1b8d525eb152a308972f +us__scenario_090__snap,8b4c8cd1fbf6e67e564afb530189a2db3a9ffa4a064ff14ac5f513cb0190daee,e2ee14fe34e800822456143b8d32fa78b83adbe276ad36e517527d0fe4abf4ec +us__scenario_090__state_income_tax_before_refundable_credits,9bf19f37bb7c98744890933cf4026b4d9ce5c0b25597dff3b06dc154f5761920,2080cd651c58a1a1cc934a4b1a1c0bc864d7648f4b787ffae81f9d0c759a9e61 +us__scenario_090__state_refundable_credits,c69dcf0865eb11ac1db95892523febda5700e6f4cf89c009ebdd24facf4c27cf,b7830c44d0e70c649f4120439b44056b0b8aa160cb07335687e64e09754491f8 +us__scenario_090__tanf,0d5de46899b23929efe04fdc55ccaefaa9ac6fed2a89f738f3cf646a16fc1654,4497551dc0871de67850eb45ec3df301fb19656905b16327354b507d2418f302 +us__scenario_091__federal_income_tax_before_refundable_credits,25abe3dab1c432cf24370f63e280cd751621416abd44eeb48568e7a375503556,65b18a70ecd45728ca9402a7b6cc7e4b4668f3b3c8f5fcbd04c038ec4fb97d0e +us__scenario_091__federal_refundable_credits,8df1f4ea06a8e5e061a9360214c3eb0d2aa9cc87b58724a5166427fce19f53c4,b93387945c5ced8c23931f580276818d481bc9c6fe3ff01595fe3a1e41976a0b +us__scenario_091__head_medicaid_eligible,09a593f97e8e0f55da018a2c9dc3bf5d71862ebadc6a1f10d828feac711e3733,bc1816dbce66b03d332116782e6b3b849819fc83821bd7f0e41c1a1afaeabc6d +us__scenario_091__payroll_tax,264502af818f1fef1c23b60840d544b0d784f247b7893139e0a5db39ac7efab9,14b1e5791e33c8a7a168e5ec27dd8eb1cb6c52ec2661a8170cfe5f14a65a4dd7 +us__scenario_091__state_income_tax_before_refundable_credits,ac9cd8333e6a2cbddf3cfb389ca1f9265fe2c238ad311954200e8db16abbf90e,b6e8b55b903170f86e341c8a3aee723b1d174d78659fb811d979c404dfd65854 +us__scenario_092__federal_income_tax_before_refundable_credits,9eb3008ac41ee85eb6a6e7109170d6762b486b8f3b1a0eab767ad1df5004fc21,a6a8d17eda6ba980a1db0841174d0f345b686bb5b24c454c5655dddccd776189 +us__scenario_092__head_medicaid_eligible,d0656f0e15f47f9043415bc39d9b5b1eef75d2cb375a0463806980e068e0192c,ee9ba0e26fa0b67c5c7796ad5a338e1ec2b0fb7d8409ce1b14bd4cb41d28df08 +us__scenario_092__payroll_tax,05ed96e35072eada4a402d80d6455b04caea25572462171004960e696c717e2b,e53420483a6035291619ac59fa234908e0101dc75891ff043783c9f8ea2ffe9b +us__scenario_092__state_income_tax_before_refundable_credits,a7150d425e823dc20533bead348bf37380d391e8fa22d3137d70132cd18c359f,838df2ec9b676e9d0ae5359830a848e99f91fd70245f608f0f9d7523b4d04617 +us__scenario_093__dependent1_medicaid_eligible,eb1c9057f0e3cb56fdf807100bb1a683550fbbdb421c0f5f236da96c28302a3c,f99246c546c6d8c8d52f3ae8c0f9072ce6f1f3dcbcfbecd58f1094abf9a61b2d +us__scenario_093__dependent1_medicare_eligible,4c98f5d5e065d92ff76f3323c7b589f75b3e80876e179c3544ee96764806944d,64ec5368210534eb7ad9108a5fc76fa26d75feafefcbf0c1042c5c17264934b9 +us__scenario_093__federal_income_tax_before_refundable_credits,2e090f4f4dcda49240d4547494268853936b62f8dde3dfdc2ccf4238633ffae5,64c3a1b7c1e9dba12bd5460cf18033a610f4f11ed3c685a1977986ef3342c086 +us__scenario_093__payroll_tax,38652b10eb814185590450f18abe2a4f99e8f38cfd8834c0fac5f92ea6c4261c,90d5369aeb98e04288cb4edc99f3d86afc3432000627f5ee2fc3f95138dd95c0 +us__scenario_093__state_income_tax_before_refundable_credits,1abe54eb43624689045698b8386a96cfde175c1d2e3a95d3e886a1fded6e1621,3bb92b597392168d3797344d6d533f5469324efe8f9bbcc246eed79e132a000e +us__scenario_095__federal_income_tax_before_refundable_credits,ccb5c9665099f1d79a8912ce862a72df12f8fb00805a0e82f82f719b93303b14,91791c2297010efe95154511008a2e3d4dda5025fabad17253f3d8297d93d551 +us__scenario_095__federal_refundable_credits,b8bccb7e5a85fd9b6635acf822bbab9961528b69d361dc678efd92e392d2f2e7,c07bd65803460c28987589426954d25cf63a87e43c627082b58df374b144fec2 +us__scenario_095__head_medicaid_eligible,6cd67e0f6cb5665b551e0b9b5c3f83076d9acae10dd5de188266ed30298a516a,257446f7cfffdd8ae8897c54584ea6b445118e09798fd32e98a0cba5dd350e5b +us__scenario_095__payroll_tax,1c474b2f24426553c37f93431a143ee7c8c107d0fe1476f2ae5f690da317c3ce,ced8b7c0a5b863ce0ca9aafb5590cee7a4a677a2848f806dde38ea6d1ad5ceb8 +us__scenario_095__self_employment_tax,3c8a6a24a49d2fc4d33018793b727659658fe8a35cfd1a517ce44fdf107b8d4b,38f626745c3acd474cf9d1892074757a174dcdef67640ee5d9f66b152842ffda +us__scenario_095__snap,098ae23c19657b353b7a0989d70d3ea33b4c9f3721ac59087b78263f336416af,aa7021baa80f939bf546fd2535dd0ae16620d3aab855360654c5db84764581bb +us__scenario_095__state_income_tax_before_refundable_credits,a87ade1b4e8e9ee4bc49463bad701d16e0fc1b8771d00b034e2876123016cf7f,4c892d51b8c3cc174ae824a90a11422666543dad45514cf32c14d40acc1476ba +us__scenario_095__state_refundable_credits,8558bf722f09756d755dc89f429284b0363eeb7a824e4de5a377f89a4fcef70e,23f674e32065e89a52cb52c82dc54bf8572cb2e40beb971381867c777ab66142 +us__scenario_098__federal_income_tax_before_refundable_credits,a38f4b835a8615fee9e27cfc0ca40aef6c0147b04557ffcc0c4ff32c64eaa212,ec0835e4644076760e0c9e99a307ca4f8278766322d505d908502bf615fd5b97 +us__scenario_098__head_medicaid_eligible,8cd262afd7a4a5b417aefbef9b8e5e0faa2c6cbe9421dbd6c4e682bf64320080,5dc5b7fe103930e61b86379bb10b504b34a78b5c76b8d16a0dd27a520c54ffe2 +us__scenario_098__snap,b70c95dc088b00f8350eb378731243909cdac4379b57e884ff1cbbc6a565e262,d07af42cc921dd613def0a4b91761a0f0e00f600f021c6896ffd9ee03f75d6a2 +us__scenario_098__state_income_tax_before_refundable_credits,3cbd357144aec83f90dc1093c44612e75515b8e3e4501df85ea99fb576480e04,1ee09ff45f612781bd76ae55b794d277525bae38f990b10edf80c9b6633ca8d0 +us__scenario_099__child2_chip_eligible,b0f31b2f454547fe5d09dc6e639c8c4e63989fb675bf88bda01f713c545a3c52,2ea73dcd67efb98b2e7d6f0f267882c81aa878e35b41a9bdc9b7a6bb04358715 +us__scenario_099__child2_early_head_start_eligible,2fa95a87c099ff8ac433c5b2b8d7624ee6144ea3d47a7590ae159eb9a79e77b1,7b908adc23d068598ecbe1c7ef1229e4109ffcfd64159586f34c4890a30acfb8 +us__scenario_099__child2_head_start_eligible,86da19ad621cf5e66d38598fe8e04c7d92e12a7d18fd096c15dbd08065d4dbdc,ceb659b95e5d3b715ef5b5a5a454e87ea1ec5201fb99d0a1078c40d4e33054b4 +us__scenario_099__child2_wic_eligible,fcbed40dc9ba79bd8a1424465919adcd4dc22bb89308647182b676f05ffcbc10,3f3394172da33877788a16cbb8deaec82da485bee529b664be1e9c2780deab4b +us__scenario_099__federal_income_tax_before_refundable_credits,fb80ff34599d7414e348de256c9f403218b26edaba12015be17d764626f203e7,18524df596840bae585909732f4096e48a30a6d36f506916cf59cb8a3bf3da89 +us__scenario_099__federal_refundable_credits,168e1138247a6005842f1b2525b8411c148b6527f5c4f41f68a4c82918aeb4fc,de41a2f2039a46d03bf52dbfa99e68a28039d42c1a273e93e5ba5f4da587195d +us__scenario_099__free_school_meals_eligible,d7d1fdf83cad78a8e90e7d354f0f22b2bd3c1866d329379ab5d1ad8e079a199a,84af914514a8e4bdeea05bbaec708f44440faa2a6975997dd5d3c4c1d7ea51f3 +us__scenario_099__payroll_tax,fb1328b0a9b09f1a008d02534a2df6ab58e48d1734efb18dcdf6e138e45726db,2256c2ad86cee224488587f5d01b2f5664a5cd550e683e180d2ed3e5f8e8b01d +us__scenario_099__state_income_tax_before_refundable_credits,4e70fc18edd1d27e2ef589ae02325425bf0a8115221edbad4fa94bcaa38dc571,5a96439af716b7d43516665a1bdcc7db7d9b3f72cda484764c67f88daa996314 +us__scenario_100__child1_chip_eligible,7ffdbc7ca5d08f46ca46ef66182bc29a8c46cd19a72a87925e2eaa4c36d40514,92bb560355412c7e0ebc905e8a42b418aad216365b267c7272ce94e55ac5f87e +us__scenario_100__child1_head_start_eligible,49c5267e1955e218824edb896ef6afe8513c15900fcb3fa8162d8464e1d1dbec,a1bc7cfcb12a742bb47862d3d6a680916ba4dd02badedb72075603aea79cbbd6 +us__scenario_100__child1_medicaid_eligible,f4d72f7e9ec7a2a3c872bcd432b325779ba1e11df4929c9e5c21c7b880ed190c,1d1a4add8b5311c807c8f9ff80c58ae1dd96477788525eedd426296fd953f21a +us__scenario_100__child1_wic_eligible,b93165ac4af96aacdac7ab4614ddbe807465a52a9cffe523de29dd09f7e6cc24,7dc92815ef7b7b2f828b9e4083f18a1a2bbf182180e6ab46847f6c909bcdd828 +us__scenario_100__child2_chip_eligible,fde0a7af991c6e5e3740c523584c06a4a43ffeaed5c9b6febd2bad6d9a910429,b09ad5050f3a921a53a2e3185ff5a6d41efcadbb366b85b3b026ccba41247a7a +us__scenario_100__child2_early_head_start_eligible,c293f7bd6ec6d82d1698e3fea3cbc42d8e812b9003cedaa382769b50631a4924,37412541482c0bb137e2db632405a91b0b55bc15b4f2d40b00dcde3d3b285d72 +us__scenario_100__child2_head_start_eligible,6fec1a2f819664628af449323dabf4d3164be30ac606a39c9ef58fa9436970a3,d12046b335644c4a08ae44c001a1759f6910d7f63ef2ace53e15ff26574840c6 +us__scenario_100__child2_medicaid_eligible,fe030897de6850c05bf4ebedf681a49e2c56fa9d5935b651d2702155c36599be,231902793dbcb27611ef0d54fbe06ded22d5c10211053898d714c0dfcd97a12f +us__scenario_100__child2_wic_eligible,34755e3b9718b10316b26a50b9417140686100682ea3f752cb3c65eb234f96e0,163409e6e0817d44d1e1be2755c836b6154e25357a96be94c065a16cfd557b96 +us__scenario_100__federal_refundable_credits,dedc78aa47bf294cc1f15018fe354974a0d22a67d0997d1882ddb3cf3c1868dd,a9babaf7cdac3da5f128f36705badf7b3efbaff1f8e94ee9e88886406c5421fa +us__scenario_100__free_school_meals_eligible,7228854ed21c25c78f6154b660ba69b627b06c3a83ee5f214927e83d16d897ab,346b3cfc914a2538ead9b42441edb562c301cf7f59d04eb601c3825a759fb31c +us__scenario_100__head_medicaid_eligible,48e03b27b24ed322a410324bbd575170ecdc529b7ac7a62452345654b61918c5,b517187637972fb7d544a8f1f29773907108f9f95dd54d08474954e9d2bb9b02 +us__scenario_100__head_medicare_eligible,ad947d335e7a019c438dd97f875e7b9e58b25aa08455f97573282a67e2dce0ee,0ae21bb28543c90f5963805ea426fe4027fa5b581dbabbc47360e3592591e49d +us__scenario_100__head_wic_eligible,cf153d90fd04487b2e320b91ddd524f4f43a7d4486f8e9d5b7d4288f2cc16b3a,a419bd2dbaa28c22cfbf65bb7e9660ee4fca6d7157e5c153f24f71de0416eaab +us__scenario_100__payroll_tax,f864eaf0d9dfe4da779598c661c52b7491350ec4a6528e4730514d81bacadf61,2ffede13456c8f1cbd057431c0494a09c92dc28c6391cc1c6d40e249bd2b6a90 +us__scenario_100__reduced_price_school_meals_eligible,f48e0f5e3e1be80d313f5e14147df9a8518ebbade9afefaccccf8a085743b7e2,52a1034f6edfb84b8ec945fbaf13dfbbca8de2b289716d76054ae88a909c9364 +us__scenario_100__snap,5546ec301eba6b35264477f4f5cbe928a1fa521b5ee175a8e066e8b354731893,74f15fab7d138af0040de665895e5ef17bdd031bad67b387495112ceffbfdf55 +us__scenario_100__ssi,09ed7e493c8ac49b0075113658c5d540283544489f4c4a4cc90936015b747c2e,ab0f911ffe235e01b310584be8ba7b9d9e5d3d5d5a63bce6328ec0f604057f76 +us__scenario_100__state_refundable_credits,73a92e3143934b51d7fc21569bae2d19e4cd1ff6cd3b9fde63771b666ad05de4,19c2d6d31341942983ad670f4247af19ea823faaaadd7993cc622516b59abe2c +us__scenario_100__tanf,c70fb95cf994c9ce31e0bdf5b07096f9b3f04666580b32c6ee2ad6eac2718036,61106f7a525f1e786079128133e9b650038d4ed0b14f2371ec1a7f69cc35fd4a +us__scenario_101__federal_income_tax_before_refundable_credits,ed69208c41533212218261eadd9127effbc19334becce883d0b24af3cdb87c09,6ded4f3cf867cc12816d956c9344febe03ca1b4aed9b2e0ea24918d6d26ad3e1 +us__scenario_101__payroll_tax,bed608f5de9caef1a933f0990d62a08367b3615af714ebf9aa0aa75b6e3f1fec,99fdc0ed8e493e4a348c42dc035013c006a9c0d4463c25e46c7c3382b1d97e7d +us__scenario_101__state_income_tax_before_refundable_credits,4a51ce741f9c4f5e11debc391104c9bdd78ba23a379912d224cfdd8232835533,2c03e3495e68b19fb648a5c21134b23f0f35401773765bad122d24d7f9d3c2a6 +us__scenario_102__federal_income_tax_before_refundable_credits,1f4857f021de2c2cc2a579256a00a01f633850229b58de0d9df7092c5fa2b928,8c9715481ea35eab3a5881a01091de3540da5a1ea5f41de266a0fe8024b263e4 +us__scenario_102__federal_refundable_credits,1e601d03532140264686b284f460d2116091b602a6b80a4ec30d7f3799324997,ae74d7f50b0d6de89a090d1dfdeff35fc4953aa940e1e26481f55c4698c71f54 +us__scenario_102__head_medicaid_eligible,d5d3ef6446b0a066d66bc5785940e16828da8116caa2149a1d1927dad6a94729,751f23df5475cd27a58eb4a61a8afa5c9cbd25d9c10b6dd5145119c1a4c11c51 +us__scenario_102__head_medicare_eligible,e035d850b7f1bb7e1613eb438e8df66b7425a12cd160b2a8f7c8093a49417457,e3c7848bcb73d4b112379903f96efaeb0a86c33329271f1d002b575905cd1680 +us__scenario_102__payroll_tax,9e82e5aad615feda16e19f80961d7cfa34d536d6115c6f5684366e8b8a984de5,49094627773fcf98ec5b1c460d43f8bddc2f72e90d5dec41ec2daa4a583f3605 +us__scenario_102__self_employment_tax,8657d250c36b165d52ef2648ecf991d3aa4517cae167f3a8f673b5cbc1b6e573,c20e61a529c60cfd66a8ed543ec7b930133dc678aa78d283cc6d43b1eb7adccd +us__scenario_102__spouse_medicare_eligible,f46b896f82b5523fe3003eee2b68291323798151d9818d1b614c72404193c311,56555b70da3ddbd5bf3f844126541b7c4ec63e69b3415549dc4e9ac73da019ad +us__scenario_102__state_income_tax_before_refundable_credits,274d7a14d83b6264b2c437f0754c09843e9d2f6c834a67d5331077e1935eea38,67b37ee4f21d701c7986e8bf3f51ba84db82ec6f1d782b2da8d22cc989b5439c +us__scenario_104__head_medicaid_eligible,2be536aa9e33fdc74fb76b8cc8ecb299bc6c6c4b0ec941b37283379e4c02507d,df565d71c1643211ff3ff2db8de166d96e7816be78baf39c115fb8696c9bd35d +us__scenario_104__snap,ddd90e125c60e639aab3fb91979a33aac4d6de88fda09ffef7bf32d89e599952,918f3d22f7bf8d30f940c35441a4d60dae9940efea8118f52f0f5f7e7f83285e +us__scenario_104__state_refundable_credits,e1ccbfae16f82ecc76808d1bdc19607ec277f01a04140777f2853653d7624c9b,48a4b28bcb0b63f57af2e47109f0cda96823db8e281ebbcf46675e98d8a0f605 +us__scenario_107__federal_income_tax_before_refundable_credits,ae86382325d449aa954d9ac405e3b35ed09a4f64375a3dbb9b29fa82e4b702f7,f392af050b61548f936e52455de3aeefd5f32310a0f1478d8d6d6381b054ddef +us__scenario_107__federal_refundable_credits,8ce30c4292c1d550874c94f936ec81c91e7bda6501b4425baf686768ce12bc97,f4bebed066e53a96dc3758b7efd30b421663fb3a56019d93ca6b674097b6b0bb +us__scenario_107__head_medicaid_eligible,d64333207d87b423bad3bae0ef46495e35857778fa0a6c60a1636ec8f557f280,22e27326f52dcbf196f98ca50d3ea3febe7691655708a4c561ab7dc331e6510d +us__scenario_107__payroll_tax,492359a5328a9233be292b09edf7a0011778a0aa21744aa6abb5a771d4cd32ba,45a6e6efced759812f979e51794ee4447950fa7d3a99b59b2234081e1a202d8d +us__scenario_107__snap,a8c76325a7f15ddd29f9480f8ae9a9ed424bb6c0b9be982a75296f8a78654c73,c29da5cb840d5ef24525b4df767de20d35334c4bc5e7a8e0b1c0fe17a2ae5ad3 +us__scenario_107__state_income_tax_before_refundable_credits,3ca54642b83a48e0a9c810433c9d185b443b9d4a1ad29176d771dd79d55a52ae,e4ed77a3fbefcda2d7931eb41de980c5e87f9b61f40a9b0ef77da25bf775342e +us__scenario_108__federal_income_tax_before_refundable_credits,f055f23f31453e1f51c7f34882dd643ddb36ad83b714f8543067fd2c4c9d7b0a,99801a44d1aabbd23aa38a7d1b29d799f4260f88f37cad7feee5baeb7a62f45c +us__scenario_108__federal_refundable_credits,1c413f0dc2dc3bb0f8b33e042c2805e0966026e9c417ee210620d33856d14446,481c7d2928c83010123aee24ffd4f4135a74a91a3ccbced7ae43aa06a42e3d81 +us__scenario_108__head_medicaid_eligible,db5a96055dfef40d51d99ae837e5b5992280904d4fa813876ea34b4533686a91,ac8f0f7044564eada7aa064f3a30c5e854fb06c21936ca03bd2b096986840fda +us__scenario_108__payroll_tax,31501e3f7b6106c30f3e88cd461b81b1c25c0f2778a111b77734291864028f81,07b5f9a7fd9978bfd4172f80262fee544dbb2b871f1cb1503bcef419707fd653 +us__scenario_108__snap,b3f71e02d3c2ce99092eb763138308233e74ac28a9705b7fafe2f1b1534c0e38,ff1af8ca8475b1bed5d17a3af7e079f2a0c18fe1424172b5586c47b920baae9e +us__scenario_108__state_income_tax_before_refundable_credits,e9c0825df31e9d945d395d03ab9ebcf9afe01dfe6324c28d93a91bf7e5144071,55578f41ddae1c0e2d0efa06856197efe8c704de227a90f7d757aa77a64222f2 +us__scenario_108__state_refundable_credits,b9519c04fdc693206d8d5778de6cc014edfb68d02afdd645679fb9ab8bce80e0,52e48492b54957b7ff7b983bc959cb1472a5cdb46a5f115baa78000641a9f88f +us__scenario_109__child1_chip_eligible,95b07ff668b9575e4309589318f19b58020cce8d40172ba5de626fa839c9caff,4179653b3eb573f877ae4cd6dac9362b5baee7f96f5901830f8a367ffdc062de +us__scenario_109__child1_medicaid_eligible,c806fea0f6a8b994fdf9205258f536acadec10c25daa12f6ca105c8e4034bcf6,4bc3938fd7c76188713048b6a01cf5ec34245afb71c38abf6bc332264fbcc4bf +us__scenario_109__child2_chip_eligible,7e7873d309632b91ea07dcdba15c95c84e30759c00fe7be071247b983c2f9e8d,8437126d76f27b635bb201c035b1ebf73a8df5352962e49c1237d49317570180 +us__scenario_109__child2_head_start_eligible,f46b048bee44d197f028cae7290c62e42ccf3e5e5e7468331607ec3c136d4119,ff4e4d4fac95ae515a1ded072a26a93678a4fee677e318184fa805fe031d4f66 +us__scenario_109__child2_medicaid_eligible,5f41323210a69526bcef644dc26eaa0f5d4d3cab8aa7dd7adc42d0cf28e34f55,0deb8c46388d3597980907374bd27e0b487b907c9945a868f31c08202fa75d8e +us__scenario_109__child2_wic_eligible,ff0267c6ef81551f3877368a43b5218cfa245e1194a95dff0e69ebf70d8ccb77,78f286838cd944b3e37925f586cc2f2f5f68861204f3e5dc8be6594c03a3bebd +us__scenario_109__child3_chip_eligible,dde97cf5286042bf287f0a20fb76fdead5508c0e9a84b29c104e9bf8ed5b1fdc,b30f8127d5631222cae718b98ada52886c0a400050b5ba1cf7e3bdf1a186010e +us__scenario_109__child3_early_head_start_eligible,2d40b21ecfddd39defe7add5b274ca5d541c7d4ba5600128b0631eabf2eff666,26176a8ad6de724f4d8c5c47e4911c55b6b4e8daca6cb6cbc98057751f501741 +us__scenario_109__child3_head_start_eligible,9bf66fdc97e155bf72a12f9b4de76f3e3ce2d76e39610e67bc6c10702f25e6da,3652c93d43605f903e1e3ae9388150d47a97f2672cd617fadc556865b92668ae +us__scenario_109__child3_medicaid_eligible,ca2be8657ff9de56b93cb7cb0a523e8fb5c22bea9e253d94661966a0a6bde1aa,f0923debc63db7e5edf9b40de1a35df8937ce0e99d3ecc27fb60c6ef3b8c8b51 +us__scenario_109__child3_wic_eligible,af474fc81edb2a4f723848479c0f3a499f588f0081e83740d215ba59a19cc3ad,9ccf3a7e8a096d6450d84dbc2fda4136ebdf18be87515b08b2f0f23e73a134ff +us__scenario_109__federal_income_tax_before_refundable_credits,4d8d528b04dbf78859d3da734e6c564f21c7ddfb57d6cc1cbf6c8c792771db53,be4af2f46c406bcb08c621ba20433a734d9ecdffb1a47b5a2973219c401cd9f5 +us__scenario_109__federal_refundable_credits,acbdaf149b8a09bf55f5c20dfa7b12277165727c3023b7e8f620335d331c9e18,02d325ae1ee91abd23a80ea484eced2a6f169ae98a10eaa328c6de768434fe4d +us__scenario_109__free_school_meals_eligible,de534fa64d3370e65f6d7b5d407bdfaf4489cf1e71de949d28ef815da2073b76,b92f7666fda454f815d74ccfb926e9e5fac2e7ce754fe1ce9de630d979ddf3a3 +us__scenario_109__head_medicaid_eligible,e28e69e317185feff2783d9cdd3ee1a87cd2afd9d26cec04e2be619b20e454f9,4efd6c417a20ab25a4bfe8d33b538e37264cba3c8fee877377c66e04a410b8c2 +us__scenario_109__head_wic_eligible,2137dfc79fb6f44c4be90a4c29879317924f7d86111820852a78f38605790804,e9f69a0cc76f50db8b403dd46f905776815a156a2bbe7c53a50003ed22591cbe +us__scenario_109__reduced_price_school_meals_eligible,31bf934a56c29f7f00d7c7c42f59c897b5a175a61da73e7ae5fe75fc6ecc7bcf,6d9b2b7eaaf2e304e0ad51c254f0a181099931d157140a5d55596d84ce84fa40 +us__scenario_109__self_employment_tax,0133f3dee663890285aa94fd9005cd18c50c20487feb9ea9490901e54c34cd07,38c60acdc7d7daa6dde74f177afa29c26a7fd504d2bebe77f40a0527f20ea4e0 +us__scenario_109__snap,3e5c7fb6d09f3c8a811a6c153ae57b5245e907bfe126c17ee48929a688320bc7,9b7a7dbd91b86c0989d9ca17060ed9858a1f75c6f62857245ff6c4cae51e4d0a +us__scenario_109__spouse_medicaid_eligible,f21941a8c827445dc9ffd1d9fec946c43e531df7ecb97925148b006aa396d9aa,f1e03b0784dd96fdb9eedac1f34427f2adde5f2082fd660972d62c88f59cee88 +us__scenario_109__spouse_wic_eligible,325d38a414903be1464707ece33fe114afa63a8e00c8b52f57c34034f5924398,c5b9e14bbc394fd2f65c3b4ddf1e3d8f3c0135764e4f9125c0fe5b8a07742499 +us__scenario_110__federal_income_tax_before_refundable_credits,68fd6bb9d44c315a4d869603a9ffd143e99b64430d2a4bb66db58868ddec819f,b8e7ffd62f3b7a0ce5f9ecbcb59adbd9ac09f97502b40b47b86e41b094947765 +us__scenario_110__payroll_tax,abb5e050dded944da178dc2c39c85ad041fceccd4fa99f5374a9ce6d010f4f53,991c2b3ed25a7714261e4df135f08e750fe84fb1af72cc83aa2849cb711f8969 +us__scenario_110__state_income_tax_before_refundable_credits,43aad489d0c1a53b8a030b85f0aa0485f9eb77cdf019ccef3e3e4843f135cd7c,c0f68bed3abfb94d9eb9887ec8292a0b8a0cbeb761ea9841c8cab96a106ec9df +us__scenario_111__federal_income_tax_before_refundable_credits,43d0ef06a041e5007925f42ac571a195b241e9637d19d0b8558604819be914b7,492337b58b95c3ed7954c8ca550c3374f2bca1fbdcc4e373d055964e51f0c9a8 +us__scenario_111__head_medicaid_eligible,a430153ce573528490e51ba0f40315658e0569a761c172d4fb55e336e007df14,f1bcbd2a52d30f24f2da19e19df059578e4b017e943372f763c34343a48a5bad +us__scenario_111__payroll_tax,3beb0ace746fb3a63ee03e1205ad309dedd3bbd43062ea75a433d7f111dfb361,513c770f6a6cc764c39ac5c53c079f27692b9fd3f138e0bf9692c319d16e5751 +us__scenario_111__snap,b64d6497d900bdfaa8402f2f8666d20127f340800b74e293d0e51f83d6d38fb0,95a092d05068cee4aadef05c2d697174c94c9384bde66fb42058ea747074461c +us__scenario_111__spouse_medicaid_eligible,10eadd161726da6ef4678285641680ba6165cfb04ff1f3e87be611520f7ed144,b99d3dcc48b45ae2fdd187409fc6601176da2db82907c6abd08a8228e4e5cf65 +us__scenario_111__spouse_medicare_eligible,1b4acc243558ec551f071bc68f0a9e2db86c905b08c58e1405bd372e454ac526,b6c11aaa57d23f567b259433f8a4b138419cdf7161c6d5d879b457aeee823d04 +us__scenario_112__federal_income_tax_before_refundable_credits,4e724060206eff175e4504eb2caa1b272013bf1c7fca80397d6217b9f3288157,0ecc7c6bee1489e81a8eae1183d2d9b50b06f4b4da652a5873489d38a9639c95 +us__scenario_112__federal_refundable_credits,95a3b68ff7aaee81a894f49e8cfbf49f9874b2578840b33e11a3d0cd65f72814,f9e281d4620ae229345fd73a1a1d3cb88c1f5ab516fe82141043676f97d74bfd +us__scenario_112__head_medicaid_eligible,239be8c8c005b939b31e41b63d3d9c094c6a301e60a93092e3ac7af65a2ff275,cf005069f379d417e3274be065641b60c14779d6e1817a078b5769f98c136096 +us__scenario_112__head_wic_eligible,c3991638780039d33fe9f6e3c34d50da701f0c919f0f67998e62b5b53d596411,a67037d1bf6467d0560a3fa3d2cf93a456e9ab93be0965f511a08e78413a4989 +us__scenario_112__payroll_tax,60d96684620fee48910f39733de20767413c14547b1142fb9f6c559244bc7172,62ff4ac380676683da71168ce441119a931c332f23a94ad2fe38deec48b691ef +us__scenario_112__self_employment_tax,92f4fde0e650ff25da9e097ad237c91b7c893d29962e071cabe9ebcab71ff218,2fffa4ac34206dd5c70328cefe0d4856b80f86cf7cd71b7f441abfe294c8f3bd +us__scenario_112__snap,379654f6324baa1470449fa2a120c0790bed2a3d81a702fcc7ae5977673a12fd,ac170d3f4cbc670f941a5c58f3cda35f1e513a9e2cd4ac13666f93547a79bb25 +us__scenario_114__federal_income_tax_before_refundable_credits,d8becc4ab482ebf61fd3b9f3225734b03480be064e2f6ac852f867713bc6363b,1330438b2d1f63b43b908ba931b410f08e1613df5e3ad00c7e8f9202f7d6975f +us__scenario_114__federal_refundable_credits,4893ee696228cb54f450e9dc6c27be2f269016891f0fb00d235926b36107118d,67a85e18b6df494d0bea9ea6322784dde7da0013dd82731b1ec49d2b48906f66 +us__scenario_114__payroll_tax,728ce80c383ebea347b59f6b25ac22a735cea327de9586aae7a1f47047870fb6,61c5b2d82967d3af7df146d7ea4412e0007c2937109a90087177aea3b7f9f9d4 +us__scenario_114__self_employment_tax,976e0494715515829e2ad02f305cc25016077051b55baec3eee1377956c3e8a4,b28a5d2fb6758654353e8be605f64d0a843182a0a08e65b3595e4fe63be5e5c1 +us__scenario_114__state_income_tax_before_refundable_credits,704f4a71f89ae1cc31c22e019aabcc6733ac38edd6ffa8d92d8f3527fa8b86c9,1324cc2803927a67fa537f5370bcf3873c2261eb9318842bddee4f47469cc0ec +us__scenario_115__head_medicaid_eligible,ecce7759f91e86e416e3790ebd94fd8398dd43380b496474d0517ff5ac21990d,02f548c330509d4f280a34d9e2f7b2b18dfc6b767dce5a51e0e17865eebdfbce +us__scenario_115__snap,634c417adf074bdd41cf1d4360f098e38645492ca5e486e05ffd1764bf24f9ec,3ede57fcbd96772eda588d55904edef6c1fcd3fd4c276c5e86df23367026ceb2 +us__scenario_115__state_income_tax_before_refundable_credits,10a710484b385b25501f9c36b0efa3a056388a2f2a2f4a4fe871121f10b2f7e3,aeb8191318a24758a799134d73edfffc608d24b7b974a76f3af488e96cc01a24 +us__scenario_116__snap,d875ad46611cd5214dfaa9c96d4a3a8821334ab2c97a93394607f17b279f7d8f,6971485bb9b884f5db2b4c85b8031d6f02b12efab642ee87a46ddd72a24b436b +us__scenario_116__spouse_medicaid_eligible,c61163baf718458e30f529ab3f9d84f8c6fcd32dd25dd0debc327713d83b9da3,c71899c630c2f0e9f6cea6d20f3261faf36b427cf5d70c0d32395f1b4e481f19 +us__scenario_116__spouse_medicare_eligible,0efd2eec8758157d1f455e443b0f698c82bc22bce2698af842aedd4040c08641,5d953ccfb4800970e84f8a0c3ac761057cddb400bfe906d65761c0600b70d6c1 +us__scenario_117__child3_early_head_start_eligible,0926d12e2f0241120078e26167ad9cf017333dcb4b8955a2d5a09bf892c007b2,fb8a3a1900d19278385a97eb392b00206f4d61e82f090c754abb697565cb20a1 +us__scenario_117__child3_head_start_eligible,44703423e85c5569457e69aa31ef80b2659065d6f27fe639e083fbebf58fe5ab,8c0858c6f18e6a21d8b2e443a088a19d25ab0b0c09c0aa10aa25a40a0271335f +us__scenario_117__child3_wic_eligible,31ff794afa1cb3ddb82ead5b6dece0a9b998b02ff644f1214d827f7b6a90d20b,4c82af05c92393e1707d1699f64d3d02619c4a506d76d10ba2ab4e9d4f4165f1 +us__scenario_117__federal_income_tax_before_refundable_credits,09c0552d4e79e0ebf6414adfdbbdff8386bc4ccde56909916a15cf797970ae4f,3162677f98080f94f6cb0728beb1dec62f5d1db1c2e2802abff91cf51414a2e2 +us__scenario_117__federal_refundable_credits,b0f1d7553d95d493cb21d918e67ffda978fea56f3db7a5f7c6a48ea50230606b,e7d2a782ec4c5074b9650ede908b9a02eb5cb838fddf78745e839b539a6b312c +us__scenario_117__head_wic_eligible,d65f67cf6634c591eee1aa2dca4f3abb6eab181efbfa9683cd69121f74d78ba3,999b453c4ea5381c95feea026b861503c86c28346fea8abcefb8e3f65a6bc3cd +us__scenario_117__payroll_tax,c335e18898b96ffe23ac22315159e03f576dde59837f0d53c1823e526b1c7c16,0e50359baa35a7eeca90dda0d4e5001dc4174f82e94c4a7e9b95419c680860eb +us__scenario_117__spouse_wic_eligible,3de7fecf4f8871a549d78944fc9fa629faad351400955f029091211d2cef52a5,d3518137770e1e43fafb36317e5a4dd1224f90d663bc6e7f03944820d34a09f9 +us__scenario_117__state_income_tax_before_refundable_credits,db635693d7bc860de41c06cc4ca468a09d16932913c718264891e5b2b8b9042f,798a891a9d7930043fa8daa6edacd7a9b0a55d6070a6d72dfd4e75386f3df08c +us__scenario_118__head_medicaid_eligible,e4ab3e94a6988cdc765f69cc7521cda5d6deeb91acd4732a887561f9863e2e87,4b659d697f6a85093b208c45914fd9e67cc0c04e9c9352bdab973ff2d31e5a8c +us__scenario_118__payroll_tax,ac99d20fe15a935dd5b988b6dcac640b33c3845851e44786a260727e14f52c85,d6aaec3987c5199e4297fc2d723d03dd240caba24a4f9581a1fbe2c170d6f0fd +us__scenario_118__snap,f90f975a772bca6957782b86a5f23dc2496c75cc37ca230676f26306de805014,9e3071993def3b401623d836b3c5b9f372ab908d7dc02a1c5a5d32db88d5d858 +us__scenario_118__ssi,294d6c5c9dccc91c2f8137be0f4a8069dedbaee62f2ebb4966070df98d3305d3,229093fb0cf25b2ecb4477f957041bd7aa80cdc7140bcb1f2086ece5a9ef0a30 +us__scenario_118__state_refundable_credits,73de087579ec693e6b08083dc403790ee946b2b2e25c15a1914b88ecca39c9f9,3f4e0f7e3d6625e5e4632e0e16d77393e6e264c58b8aa5511a853fc5ec2055f4 +us__scenario_119__child1_chip_eligible,380eef556ace7ef7f6bc5d713e5843dd735ad306514c576ff1057355704ad129,17ecdeb200e5875fce56e5e12f275c3330e9e62b756e4953dc2ed5069928886f +us__scenario_119__child1_medicaid_eligible,f668ff1e7ae90feca959db659a3ae7454694f0086f0cf96c21aa48fbcd3c37e8,d513b32caf26de5ce33d49f27064fd8be3795cf85bab12eb5a7c2ed57c7d4ecf +us__scenario_119__child2_chip_eligible,a9b1e54d772c0f72bd23e442ebcb3e4b2c22424dd8e4b30ffdef6359a3ef7d26,d3145c69dc8724c0cbfca3788c092cc91871ff062710a5159c697e1e49979ab8 +us__scenario_119__child2_medicaid_eligible,7cb313e5e8fb5d9873baafe18f7267f69f0fa26462027805052b82945c94b108,1186a23be3c19588241bfe8570c02e655f77fcb020fb966268ed6272dacfb313 +us__scenario_119__federal_income_tax_before_refundable_credits,68f748ccbc0a2fc3a151aa72a49c5df3513a500d7daab36adb6fc6f2384abf06,483b63d5796504255a16be834b8d108b39917f05126887cdeb0a6d581eb8e86a +us__scenario_119__federal_refundable_credits,c86b4c5002d60ef35308098bc91028e3720d2fe256cd937d01a8321d4b78e117,cfa26c1a66e81f4b3bcb62267481ecd63558efd4d9d79e3937e53b9096828cd5 +us__scenario_119__payroll_tax,2e3e6e070681e314886ab9258066fffb00046304cf84cf51d7c7498a5412ef6f,e5294797ad9fd67690c33e56af234abe913eba73bb9abb5701f49a53467ac8b0 +us__scenario_119__reduced_price_school_meals_eligible,71c358e735d0e5b84b1fce88b1b2c6e2fd2a4a7aae846d4ffd6292a0bda5f688,5ef0c626bbd708c1ff5dae3ac9d239f8441037088dc638f482ce246c6ff6e556 +us__scenario_119__state_income_tax_before_refundable_credits,08c3b45a7808144983f148fe5477465a75d979819c6edee3037f3313e9e73c7b,ec826f2d7459dde71206fc346cc4f3cac0c54c68e2bb00297312d56f39b2a444 +us__scenario_119__state_refundable_credits,c9054d4b0d62e0ff8c80ef82f83f04616f35a2225e69fb51135c74de939ed838,20f73fb637134300b1c7d3a74ecec6ad9a4e2544163308c856e4d6c153c20603 +us__scenario_120__federal_income_tax_before_refundable_credits,6d5a80ab6f4dc9596c6bf051a1e79e6e54a51e510f7bf7b9c1127e82d16a7255,85b90ac023b424b1ded692bdfcf1264c3057abec7b76a9a9f5e969be2c1d7f1a +us__scenario_120__head_medicare_eligible,38423457f0d2f776538ce275d1e8d4614066e17bd7c163adcaf72765980cc522,eeb633fc78773bcd0efed8dd71893b20fa0213a5605078c0a2c01eaa9bb128c8 +us__scenario_120__payroll_tax,2450d5a1ff69962c08526521840a45f0c7f84dae8c9387a57fb6fbe9021265d7,f593478dcaf6e4ab985215ea0a73b1b1497befaa5550aff74c5e3f079eb6c91d +us__scenario_120__state_income_tax_before_refundable_credits,f4210559b88077959775e708c92318c150ffa2a3122e3b52c0b62261db8a031b,e423fe128a5dd75e9e5311634511ace9d93026dfae1b11c6279deb11017b2d1c +us__scenario_121__federal_income_tax_before_refundable_credits,61bfca59d67cc2550a0886929d585cd61b021edbf52e10b02a116c6eab77cf70,ec63cc2614dabc2731ae4cd4c2aaf51ff02a2bdfe46e944b673d4d3db7425b36 +us__scenario_121__federal_refundable_credits,21245c794e7489d82d006c47eb91924e7adf2641c548f97a077a37fd96152dfd,172a7cc3c7dc2766ed07f83ffd2fd74afa82fa4903585a1973982dc6fef8e94c +us__scenario_121__head_medicaid_eligible,f1d2f12bbacc579e157d86e1c3ba16e03e842336293472f8d256c129adb3b4b9,72fbf4afa7aaf9bc3d5898710bac6ea7e3d7a5effb3fa7270abecb173b859f31 +us__scenario_121__head_medicare_eligible,30e015f3c46d6e5083eb214a17bde851abdeb42c68874f9786caef816b0bc938,8cef3435d06dc6ae0532f2b09b4cdb9a08690a1039a1bcb4495493fd262b055e +us__scenario_121__payroll_tax,640422209d50a59709a5d256d27a53027530bcc184a59d09704b2abe58ad7d09,66b09cf08016c1dc3370ec43fede9a05b96ebc5e009ae2756e067fcf1aa6e9db +us__scenario_121__snap,03512911eb78f3468bfaada9c89697f192ab69aad5e3e2b0dc0501a6de854b51,c27267ef789ef6c54f0a9673c990e2555ccd908b694d96d06d125f1ab6da2aed +us__scenario_121__state_income_tax_before_refundable_credits,ce60fe57a06785eb16d777ea41b1cd3c9a76e1982937a20d627a5a68c819c61f,a8b657c3e9398e752ec8ab3cf4d875ba203398c56ba07803b1a6b0ffc7533375 +us__scenario_122__federal_income_tax_before_refundable_credits,a8665f2b50ca125ae9645b91e160449da5abe95dd954c6aa347a932bd2280eb5,1bcad0e4f0b82d90018af01ef69e8f82a88b28e4c8cc57ecba9561a54461e5fd +us__scenario_122__head_medicaid_eligible,9cb1915eea284a648754090f09defe9de9174b415c189f7fbe36e396b2cc153f,f33e1094f352c7a871ac9bddd720e1a9cf13c42735d4cbcc766bb5775599be76 +us__scenario_122__payroll_tax,6194402baac22b4f36529d9b2ff9da2c38282f5954eec05586484ffb93af6ba9,8b7cfde758ff2f97f0bc27f7eda3f2326f9a04ff7ca3d277ef9ff936dbed67b3 +us__scenario_122__state_income_tax_before_refundable_credits,75f588c4d9eb58d82013b0605131ef9aba877b373fb4f090efb382180ba8f9c5,0fc5ffe168fd21ce539b89d2bd0df56c99122efa86d0b0de21666624eb2b97bd +us__scenario_122__state_refundable_credits,b032c5c37245f6c6c08e03b129af3c18af4f5095260eb2315c74fae180986175,d04072605a758bf7afdef0d7bd603251a3594b41f84b4d7c4d34118c1787cf0b +us__scenario_123__child1_chip_eligible,396ea9294f25df4757accd3fa501d5116ab243cfd339fa977f462450b459067d,655d730efd8a49a87d1448fd3a55ff88b9872ab3bd5b72cf5450350f7bc8ad23 +us__scenario_123__federal_income_tax_before_refundable_credits,82221be4970b2d7513bf2939d0aa78f1e6580349c0213fb56e05970a2515fe74,4f6f31d6328752b8321dd1bf304c4fc915568398d887aef734e362c41ac99e02 +us__scenario_123__federal_refundable_credits,1b4e0478dd57381352d13443a3bb844989a26c0843514ebb45d2a142d0653232,a9abcef52c7be874497616bf1ce520817ca82b28aadba529c84c2bffd47fa435 +us__scenario_123__head_medicaid_eligible,bcff1a25f1f72e5213b3c6611ef16ca31b2f8bf6f89a2fa9376e171df6866671,746a18bf8fd8d6650ea14f83587282a4119c5a7a4e053619f18a86b11b82f84c +us__scenario_123__local_income_tax,20e77243a50a63813619ff778a75cb454aa55599ed980836cd48febee90e15d0,42e5e539f01cd09f5ec915d9614e97528dae7209472e7d05ad0a89e15e0efe04 +us__scenario_123__payroll_tax,d21e14fb430142fadc31cfd53c286f4667e1b13cea2947101e69a8fe5e81fbec,ef1083a6758848ebdec794e2164697bd0388d395fdac65a8dc01cf22f1579f48 +us__scenario_123__spouse_medicaid_eligible,0ea7b3c8cc4c093da15503cb7cfaf83494a83b12184a598bd2138df85b630449,40d4250a45815d85b6550450c375aa9390128d15b3c74eddad4687365e5ab3cf +us__scenario_123__state_income_tax_before_refundable_credits,21d9ebdded056874eb0dfd5b9a639f75fd70fcaac85ec566f255b9a8607b781e,ad43142d632889e7c01321c24189a6c1c30fac9148f5d24c28123cb66fa7b12e diff --git a/docs/musewonder/seed_recovery.json b/docs/musewonder/seed_recovery.json new file mode 100644 index 00000000..229c0915 --- /dev/null +++ b/docs/musewonder/seed_recovery.json @@ -0,0 +1,1633 @@ +{ + "note": "Release dashboard-data-20261010's 232 re-judged verdicts, rebuilt by scripts/recover_20261010_audit.py from each judge's Claude Code session transcript after the release's stage was deleted with its worktree on 2026-10-10. Each case's prompt.md (the transcript's one user text message) and verdict.json (its accepted StructuredOutput input, written as the runner writes it) hash to docs/haiku55/judge_provenance.json. The login names the judge config the transcript was found in.", + "counts": { + "claude-18": 39, + "pb-judge": 193 + }, + "cases": [ + { + "case_id": "us__scenario_000__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "dfd8d58f-f7cf-4730-b2fa-6396a45ba1ae", + "prompt_sha256": "c1943e89a6fe22c99a2682b518825fccad30e4994d0ae5721e05b06643e60b05", + "verdict_sha256": "f1736832f26c9b0463e1b0c9401ea84592d833af117e3da1d1f2418a881b27c0" + }, + { + "case_id": "us__scenario_003__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "4bad012b-dc97-44a6-9017-2f6f8ee44e09", + "prompt_sha256": "8c65b91ee004b854a0295b8aa0e7eb2b147a7679d98ec60f97e187c116204aca", + "verdict_sha256": "f67e7a43fee2e9f2abd7e3b8a15a1a96d5ad19f945806f1e62918acea21899c1" + }, + { + "case_id": "us__scenario_003__head_medicare_eligible", + "login": "pb-judge", + "session_id": "7e17c23b-e9d3-4465-98c8-13a179e98f4a", + "prompt_sha256": "72023d07ebc68ecacd9082cfffe74cf677f04e55c5e8ed763c05ae96a29acb3a", + "verdict_sha256": "23246851850f9d97109d8734aeb248e05175cfd13112bd29e3601b03874c48fa" + }, + { + "case_id": "us__scenario_004__head_medicare_eligible", + "login": "pb-judge", + "session_id": "911bcbeb-575a-493c-ae84-1f4f0cce92e2", + "prompt_sha256": "346ae03a8c5e9938a30641ab9f4d6567cb6c71c998f6a16872285a56885e5c8d", + "verdict_sha256": "6a0d3eea13051465988513b7ee8aaf0b1bb0cd977dcf6fc4be3d7b538f3fa4a0" + }, + { + "case_id": "us__scenario_005__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "bf6eeaf3-587c-4b8d-bfd1-ad6b947d7e5d", + "prompt_sha256": "d5158b51c6d329c0c7b8821972eea319382766ec709db21f9a9cb1f9579f7eec", + "verdict_sha256": "bb5560621a65bbe5ac1c3db9ef0ee746ee02c57f1ffe11cccea7b0bb26675d11" + }, + { + "case_id": "us__scenario_005__payroll_tax", + "login": "pb-judge", + "session_id": "7868dac8-cbc3-4040-b2ec-4bccfb913486", + "prompt_sha256": "45a9ad0369aac5ef287f3b960ee65816107a82cf8645a1ccdf2fd62b49739c49", + "verdict_sha256": "8405c7bead7b153e302ab89b0eca5be588ef09e3679f1b961718d708bdcf657c" + }, + { + "case_id": "us__scenario_005__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "fb7e94c1-c13e-45b5-bf92-dac35a66d9f7", + "prompt_sha256": "23996f96494e704f4290f3866ccd8dd7b43ef41b528a296c43325a7d5c5a61f6", + "verdict_sha256": "7653eda025bcf5be81805f01e4374537d2322c914c1c066a1dfb76445c902059" + }, + { + "case_id": "us__scenario_007__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "470db105-20e1-4072-897c-48d9a904bef9", + "prompt_sha256": "8830d4d963659b210068f5b4bd6b82e19a132dea099f4a457c4c61b65dbec59f", + "verdict_sha256": "945362b7e820ec8b6f606a63583bd85f7e7683b46e94ead1e3a4cd913e54b6d4" + }, + { + "case_id": "us__scenario_007__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "8015903d-916f-4471-a9ef-f20aa20de7db", + "prompt_sha256": "bbce3b38ab1b15be53dae2e810c50bfed6ab82415efdab80c27c7b0995efdf60", + "verdict_sha256": "ee385050ee1a01f48f66e1d6160621e1867c15de9faac56439cf0db3634459c6" + }, + { + "case_id": "us__scenario_007__state_refundable_credits", + "login": "pb-judge", + "session_id": "b59bd07e-6230-4749-9455-6b137efe0ae4", + "prompt_sha256": "b70bdc21cbef932867199513bf1f249c9c1b1d9bcd29c36dc8627baaacd4842a", + "verdict_sha256": "f08a59b24d7dc4b1ae4ff60d1a26dede646b93fcf4a012f76ff27b04b47316e7" + }, + { + "case_id": "us__scenario_008__child5_wic_eligible", + "login": "pb-judge", + "session_id": "a5e7d310-4da5-4b44-8055-3d20649f15f2", + "prompt_sha256": "384d92fb4ddf152e224f01812c72f6040101fdc9893674450a035913387aef3b", + "verdict_sha256": "b821cd6e9f5d1675d866f37b70025149cae771fb1b89a10572c7345f9f365e35" + }, + { + "case_id": "us__scenario_008__federal_refundable_credits", + "login": "pb-judge", + "session_id": "e3b1f450-a12d-445d-9f27-c8227b242a16", + "prompt_sha256": "4e142d9c9f92b32971477c918799d6bc574faf08fcd00c728ac211f81c9dec6d", + "verdict_sha256": "519e5dc051a016eee4a14bd9aa57b953d04f39c403b8d833b147a6f6eecaba37" + }, + { + "case_id": "us__scenario_008__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "0e7f525d-a48d-4f3e-a335-2ccf0de5433d", + "prompt_sha256": "8a1564f0c291cabb2b87780aef66a5832940e45e865c7b56b5878d5de307ca33", + "verdict_sha256": "b61da31bf48c969ac3c242f02853a1d3060f8643084090da2e3ff23c60d30013" + }, + { + "case_id": "us__scenario_008__payroll_tax", + "login": "claude-18", + "session_id": "93c6bfd3-d51d-41c5-906a-8d206173b36f", + "prompt_sha256": "626c33c358a9c2ea5c1809fa5c7629c3625f5a7be799aa9ebc879c0bd04616b3", + "verdict_sha256": "9ca4a0876e65a9060128f3c9408752059315ba991f182d9aeaa705619ffaf2f4" + }, + { + "case_id": "us__scenario_008__snap", + "login": "pb-judge", + "session_id": "534edf80-5e2f-499e-b195-530923633d91", + "prompt_sha256": "62f0d8ec99b7362e549c8c146c86b993112a98fc8c59aac859615341e7011bab", + "verdict_sha256": "8ac11fab6007469a226c8eab6a10cc296b0d409c93fbb63501c9eb65bb164c27" + }, + { + "case_id": "us__scenario_008__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "5ad0319a-fe70-4edf-955b-9f5908f15175", + "prompt_sha256": "beeceb686e608a23900762839027ee423a6e2df8a9389e05dd8d0c88f8d76139", + "verdict_sha256": "c05aafb2f3e54e2037fa3f939c06d290463b53f639c63766fbb27284971732d7" + }, + { + "case_id": "us__scenario_008__state_refundable_credits", + "login": "pb-judge", + "session_id": "68c5f477-1571-46e9-ad00-cc7a803ba091", + "prompt_sha256": "e5444d0a567e1f4b8d622a8bb5c94f4d5c593c11716b024e6fe23d041f11f6a7", + "verdict_sha256": "bf20e47d5085be40b1e41050d951b428c86fe531dcb1e469a366f7e0e2dce003" + }, + { + "case_id": "us__scenario_009__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "6483f1dc-2cab-49f3-83db-634783d3f3bd", + "prompt_sha256": "243c9067e20a820bbdcafac39abeabca3fa6ece4d3316044567d7878a8989573", + "verdict_sha256": "6420da89a974cdefe8b8f652012ce92c2672e01ac659c0bf7aba223b9f96d60a" + }, + { + "case_id": "us__scenario_009__payroll_tax", + "login": "pb-judge", + "session_id": "dcbcf15c-52ea-4063-b1b7-b4f7b84ae72b", + "prompt_sha256": "34260f5eb3599bba24f24bf7c496901b1e00804a71b3b0aa310f2d1cd212584f", + "verdict_sha256": "2a9b2ae1dc24cdd9af7489f87c4aa055a417e39d97cdaca05818e6afe6b39565" + }, + { + "case_id": "us__scenario_009__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "b30443ae-765a-4398-b1e4-3018fbe225c5", + "prompt_sha256": "d7c24b29233ba300b0590c1ee58007f1cee67d4b0a401864d3806e8d859521d4", + "verdict_sha256": "d8be15d3cebc5ffa3f6917dbfde74c40b6be5efae67ad236ff761a9400058e37" + }, + { + "case_id": "us__scenario_012__child1_chip_eligible", + "login": "pb-judge", + "session_id": "531a6ef0-c794-4803-b512-ba9423c2b57e", + "prompt_sha256": "5f78a64edca0abfdef6b3607253f3c8c960614d88a5b811eba5feaef25c4ffbd", + "verdict_sha256": "20edc876c9e6010755cff47f8d4e76c021847cef4dfc9289572ea39fc56e9a1a" + }, + { + "case_id": "us__scenario_012__child1_head_start_eligible", + "login": "pb-judge", + "session_id": "3625c019-33c0-4532-b9bf-b7c4ec8bc4a4", + "prompt_sha256": "90dddff7843a5d2fde3f441347ff088c59c72968d1a9c24fa4fb6b1f204cdb0c", + "verdict_sha256": "dc2210c198d414aaa124f83a59c60d021270c4f8caae3b42949deac40a240a41" + }, + { + "case_id": "us__scenario_012__child1_wic_eligible", + "login": "pb-judge", + "session_id": "e407baa5-08bd-492f-8819-8e19ad658fc6", + "prompt_sha256": "e77cd5f13da5e8a407794921a9534f7ca6af6581dbe522575dc9b343313c85d7", + "verdict_sha256": "88629e7d905733a3ff48ae1b30fba4f2cfc067f7489b052e8df5eadf80ccfe5a" + }, + { + "case_id": "us__scenario_012__federal_refundable_credits", + "login": "pb-judge", + "session_id": "9fd401b6-211e-4ad8-92ae-c86ecf984f5a", + "prompt_sha256": "7740d513f8593cb2f0a8e254d345e4581089d545c46d79ff127c8dd619f84571", + "verdict_sha256": "5fc3c8de1d6d6945fc24f545af9d9001802ef65dfc5e3b43b128e42c6957022f" + }, + { + "case_id": "us__scenario_012__snap", + "login": "pb-judge", + "session_id": "ab1d35f1-bd37-470b-a050-2e1b3fc09d62", + "prompt_sha256": "32780900c4c47525cf9935c1e0af0dc2a0960992dc4f3a5cd1aa281a7c4e94d1", + "verdict_sha256": "adbca0f364c387b0bd7a9f70f08a35df6e71b1827e914fb6f01e824700a76c22" + }, + { + "case_id": "us__scenario_013__snap", + "login": "pb-judge", + "session_id": "7081d635-2116-4be9-8a6a-c928a2f4035d", + "prompt_sha256": "9df47c565cfddadba7768500ce9d00aab65a603c8067724499d8e24405629668", + "verdict_sha256": "fcbd4e34d6accb57de06df7a25432dae8c8edfd25c2c3ec52ea2aa9f18ebd26a" + }, + { + "case_id": "us__scenario_013__state_refundable_credits", + "login": "pb-judge", + "session_id": "104eb9eb-f7c5-4693-8aea-5d020af304a4", + "prompt_sha256": "838626f1d3518944b57f2b1efa869d0bb51d0c655f5972220bad6dcd3a0b8d4e", + "verdict_sha256": "4c6d60213e9ad24fe5f043da7bfd5e98078d355bc223488adc905503f93f8e16" + }, + { + "case_id": "us__scenario_014__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "515ae278-7583-489c-981f-3b4527ddabdd", + "prompt_sha256": "264ec8f272456c70e038bcd5c1cf16bc0b07ead1276471ed3a2f0ea7beb1d214", + "verdict_sha256": "b7519c60dd34d82acb99e8554d55a5f798a8b18435508bf9a8b08ff317925bca" + }, + { + "case_id": "us__scenario_014__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "777d1553-d7f7-4989-852a-5fa62da08eb6", + "prompt_sha256": "80fda80baa74d25c21d45fa438f9b3a5bd0008be4cc7fff8b1af0055edcb47f0", + "verdict_sha256": "2f3561012e32038914ddcfa2ad4d18501c361fae58b338a4669edfe9e67c9c60" + }, + { + "case_id": "us__scenario_015__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "4e73298f-c96c-4b4d-85aa-f108317da42e", + "prompt_sha256": "699a87518882b4e6556f2bb230f5abe18f8fad549a257ce7612a07c6e19315f8", + "verdict_sha256": "f4b21ebd85c8bbb6ee29eb6ee7004db26fad2dd289bdde6f6d5a2d067b1bbf7e" + }, + { + "case_id": "us__scenario_015__local_income_tax", + "login": "claude-18", + "session_id": "5fa9a793-a789-41a8-afe5-8c58708d11fe", + "prompt_sha256": "00bf68e6fdd682fa92e1eb11d4777e3b7fd1a3790bd888f61c67d2af9a9c199f", + "verdict_sha256": "e2f324f633297e6cfcd3832de8cba9d5a71b8f022d3bb0694dd30a0cb6e9ce91" + }, + { + "case_id": "us__scenario_015__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "2644c53e-6d9d-47cf-bcd0-6cf679f8fa11", + "prompt_sha256": "a6e0ae7db6a34ce88caed3aaa8eb27526642c281af513cac9109f5c69e532741", + "verdict_sha256": "16ef6e0393563311831d4593e17dc020318ca3291aec6112fa886a44bd3cae42" + }, + { + "case_id": "us__scenario_018__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "8c0ef917-7dca-4393-88a1-88930e6ad5f0", + "prompt_sha256": "e1f69393d34e79c43ec690c3899f07bfd7692b372c962b1ee49651722f091c3b", + "verdict_sha256": "2e31dc076b6f4cf682f88bbc9860ddcff3b37f9552d3ac4d51fea7b48d4488f3" + }, + { + "case_id": "us__scenario_018__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "70b43307-931f-4f51-b39d-695cce1c0058", + "prompt_sha256": "d5f7f7e6ab5051bfdc5df27a613778e29adfa743d628b5734bdef3488a634e84", + "verdict_sha256": "6a2f8b3a2424f0eecb3af8a0fa6fffe2947c3102d7f7317fb1d7e14b303f8b23" + }, + { + "case_id": "us__scenario_020__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a511986e-df0c-451d-b936-22d48f98efa2", + "prompt_sha256": "3d56d4656a2989443d286d4482f8e9539ed381621e968492b0a7a227beb0d128", + "verdict_sha256": "154da3be56f8792356ad68c1df2e687bdce6098354aa78c42b55df707222504a" + }, + { + "case_id": "us__scenario_020__payroll_tax", + "login": "pb-judge", + "session_id": "2bfa1a69-7222-42c0-8e26-d0e5f05e32ac", + "prompt_sha256": "b572076e3330493639c7c9bcf127beae6c01fec3e70d0f923e451aefa3ded630", + "verdict_sha256": "7ddc85df8e77c6d5ac46b2c1a720c057a447419d599f5b868e68c4e75cfb65b5" + }, + { + "case_id": "us__scenario_021__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "314e4a69-7284-45b6-b7e5-fcf7f24bf1b3", + "prompt_sha256": "0489d1e0286f1a0810e20b940add7243f7330750ea8d85eeeddc6db301fb41ae", + "verdict_sha256": "d340d4920b69e4f5ae8e4dbbf165e39677da72244115266110c91844236abcaf" + }, + { + "case_id": "us__scenario_022__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "1599eff2-d075-483a-9986-34852111857e", + "prompt_sha256": "8150bedc96fc372abc3aecd3e4dad0a8e8bddb16e52dcde7212af3f56d001515", + "verdict_sha256": "7beb1c74f6cf134e461424cb7adb52ff8d2c0248268316317bc8c1f2a3e9ea9f" + }, + { + "case_id": "us__scenario_022__payroll_tax", + "login": "pb-judge", + "session_id": "1f465602-83da-491d-b843-c864e235b44c", + "prompt_sha256": "b6c1b2c9c0d54874e6e9aa7bc222a8bdf9afb87c009c4f531b8fe4b8599a606d", + "verdict_sha256": "9761e01dbb3206827079781a3e369b1bbf7b33325d7792884ad7f36ae93bbebd" + }, + { + "case_id": "us__scenario_022__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "a572d386-ce87-45a0-9f9b-1f873c244562", + "prompt_sha256": "39d7184ce67cc967e4f96ae49fb90ab607c14e0715aab5364233b19c2bed6f8b", + "verdict_sha256": "e53852d0b72d09bd0af25189a039828bf69b14d3bf3e25d33656995c9caa5316" + }, + { + "case_id": "us__scenario_023__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "d672532a-28a4-4038-84a5-92e1c30340a1", + "prompt_sha256": "fccb23c6f0195491583ee73635d63bcb209ae7a0e0d73272115180a3691fedb4", + "verdict_sha256": "68703a97a7f9ecf4a0d1eb64dd7ba9c252cc9cb0e833d204f108846a70278321" + }, + { + "case_id": "us__scenario_023__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "1f59e478-c0fa-45c6-8796-afa1052e31ec", + "prompt_sha256": "190659f237d1dd022a966cf85802b937923c5be4fbc80f3d5a65cad7fd8927a7", + "verdict_sha256": "d1263a6c3cfb4d629b05ed061dd0c7f9d1ec173c4951232fd0f7339d5b3745cf" + }, + { + "case_id": "us__scenario_023__payroll_tax", + "login": "pb-judge", + "session_id": "5325f1b7-79a1-46b2-9365-511a9b862644", + "prompt_sha256": "38df4bbfedcbb2c96f82d22785248a55788749232e98c350c2aa975920b57eae", + "verdict_sha256": "6fbdba12726aa2d327d1fa0cb1d1a109df45ec606d008445afd24d7eb8d4e216" + }, + { + "case_id": "us__scenario_023__snap", + "login": "pb-judge", + "session_id": "4f0b48d7-21ca-441e-a96c-c9c4ae9db5a8", + "prompt_sha256": "66d7c9331499cff86a3e54cfb8e8a17332ba0be02bce66f5b05d9baaff9c2034", + "verdict_sha256": "cbd9abdb5318469c30d4aa62f2cf3dce7a5dc43d13b67553c9e3e515f5aa90c1" + }, + { + "case_id": "us__scenario_023__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "91a4b344-b61e-4b51-81b1-1fd04424caef", + "prompt_sha256": "fe8c98e1d62dbb5bc955c933c77ec4210a9d8975aa54733a3e9cd232552105c6", + "verdict_sha256": "de4c2be8725adc9b4f51d4d882d77737372875c9b323f7038260d66fea69a87e" + }, + { + "case_id": "us__scenario_023__state_refundable_credits", + "login": "pb-judge", + "session_id": "5c421a76-8cbe-4f13-a46d-5e9c21a9fc02", + "prompt_sha256": "434f2e6d5457bf852dc5cf9ce4d85af3b8b03d173b1356d9459dfa9a6b0340ea", + "verdict_sha256": "f1a357ef4a6b1adfb2f1ee07be7019147feb51a0fbbc5c027b0262cd42297d15" + }, + { + "case_id": "us__scenario_025__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "19aced58-f5d7-4ee2-8447-30358fbad5bf", + "prompt_sha256": "d9e73b237259924b17207bed8b112271109939e8def3b9543ff88eb71f24bc3b", + "verdict_sha256": "e9368fc9ac5ba87f14cd81f4810c4ffdc7b89cb10b86564b20a39cfb72ba94d6" + }, + { + "case_id": "us__scenario_025__payroll_tax", + "login": "pb-judge", + "session_id": "1c4fabc8-a12b-4829-ba09-d600f1f2ed24", + "prompt_sha256": "e745166fc15851cda53e9847d63a153aada489bd6281212505fabd8d8b8b8d77", + "verdict_sha256": "c5f274c05ca37d4762a3c18d2e5233214b4b93f8b355480b94e7ea480e14ccc3" + }, + { + "case_id": "us__scenario_025__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "edfc2694-8b3e-4029-aca5-fb0cedeb24e3", + "prompt_sha256": "3c4b71b102f038ff3c9d164e0795d1f108ac0250c783458de52d06980a6d2c33", + "verdict_sha256": "4bbe08e82d808c0817e5c8f5048e0101441f63bd6492fefceabe73cb34c02fe9" + }, + { + "case_id": "us__scenario_026__child1_medicaid_eligible", + "login": "pb-judge", + "session_id": "0d3836b7-9b45-4b2f-a0a5-e325d440540b", + "prompt_sha256": "e73080fb927f4f8fbe169491fd055ea223b49a88a9c0ee2e3f2cb5162eb88473", + "verdict_sha256": "94f16fbc73048676eeec5238e1b14f8be8bbe065f999319d54bffdeb5f9b05cd" + }, + { + "case_id": "us__scenario_026__child2_medicaid_eligible", + "login": "pb-judge", + "session_id": "2eefdcc7-60d4-4182-867d-f457afa3a095", + "prompt_sha256": "9c984b293482ecf03b4c7084746c9b025bd59dfe405e66484793dcc3d09c279e", + "verdict_sha256": "3c34a95ffbea534a26058d49ce7f05cc691e12c9be489e58c5768c86da88f07d" + }, + { + "case_id": "us__scenario_026__child3_medicaid_eligible", + "login": "pb-judge", + "session_id": "db2aae11-66a2-44d5-ab80-cb1a2b90bad2", + "prompt_sha256": "497087c1a97ea358dd7f44b2dd0bb8d111a12f931cad4c5c72adf2c672a80e1a", + "verdict_sha256": "f211d327e48f01fc7ec3ea31cc2a76d39c46ba74a463428a6cc9efa7598fbcf0" + }, + { + "case_id": "us__scenario_026__federal_refundable_credits", + "login": "pb-judge", + "session_id": "14ed0e19-e9be-45ef-8535-7e246863f894", + "prompt_sha256": "73300f41a742420ec319d8feb2e0ad5e541c7186a7cd1e21fe8f4c67e8428549", + "verdict_sha256": "e428c0318924513176b6638a9ed00bb7fcfdffc564482f711d7aaf9090ceea39" + }, + { + "case_id": "us__scenario_026__payroll_tax", + "login": "pb-judge", + "session_id": "6c4f6165-7c10-4b86-9a2b-f65fa4c193b0", + "prompt_sha256": "40a7045cc13f30876297c0f3a86c8f476c96f55c5c4131243fe42af54ed43149", + "verdict_sha256": "29363ab09f913fa9f50a6d4dbe134cd4e2edd8d4f5a0b0ab13f85bbbab28c2e0" + }, + { + "case_id": "us__scenario_026__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "bf16d2d8-72ef-48fc-9973-5518a935bcd3", + "prompt_sha256": "987887e41b1a4987bb011ee399f447e5aaaed7cab48acde16ab5954c5518aff6", + "verdict_sha256": "fb0f952f37680280f30fff8c21016479b68bc28806c48dd89f1bbc6f7cf7f850" + }, + { + "case_id": "us__scenario_027__snap", + "login": "pb-judge", + "session_id": "595cbf0e-18c3-4e74-a8c0-bc221652a530", + "prompt_sha256": "a03e0f1722da33adaad17d9171b40c6832855c5962e23c2d5d86612d99c076d6", + "verdict_sha256": "97dabd25fc0a852d8ee88a23fb2030af6fc4e601f5805b391eecafce1f47affb" + }, + { + "case_id": "us__scenario_027__spouse_medicare_eligible", + "login": "pb-judge", + "session_id": "08cc10df-6122-4e51-8a04-2ba5d4cdd554", + "prompt_sha256": "2c1891cc3b9ce6592806b394125c16e3c792113360ea4f76e53f915ab0b1ae7f", + "verdict_sha256": "e51cdbece42523623127b0d4e060891d12ac0b0f1e69e2f6b6893573492ca2ff" + }, + { + "case_id": "us__scenario_028__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "1a212d95-1481-4f86-bf0b-81d69eb59754", + "prompt_sha256": "9a5de242dd6c7bf8259c07aaab1c89c062a9354776c7240844a0b83abef923bb", + "verdict_sha256": "e636232f8ccc9b3bda2be5bc1b0f1d424247db0b239a02bfe1c407b74e980149" + }, + { + "case_id": "us__scenario_028__federal_refundable_credits", + "login": "pb-judge", + "session_id": "05f93065-f72e-4b01-a630-518829ce8b62", + "prompt_sha256": "b7250abdad7765cd103918677bad76f8e36138968b2c3d2ed87750e84af43525", + "verdict_sha256": "703f153771007ee495217db0a2b7d9ed41ee0ba9b0d4bc1919a8b784606eea5f" + }, + { + "case_id": "us__scenario_028__payroll_tax", + "login": "pb-judge", + "session_id": "2eccce1c-d524-4a6c-a9c7-5c53f32552b4", + "prompt_sha256": "50e43cbb3711731d858f6f929c1b37d701de64edd47aa183e6574e16edbf8371", + "verdict_sha256": "09c1dd3371add125f78ef9586396b5b6bcb45120cc27d3d182d1a9f291182f65" + }, + { + "case_id": "us__scenario_028__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "7387d7a5-799e-455d-aa34-a1ec9967bf20", + "prompt_sha256": "967b05135d442fbb84bba54cd1c30acdd5f5942525f9c8f4c78b79d3d4ab1901", + "verdict_sha256": "4080dcb2782f207dbd5b93bc39f4aba03142ed9c22c9480c1a134941c3a1583e" + }, + { + "case_id": "us__scenario_028__state_refundable_credits", + "login": "pb-judge", + "session_id": "05592832-ee71-4261-82a5-e02764d3bb25", + "prompt_sha256": "2f6c81ded6bf2b951bc08f32e1def26c47899e08706bf6c95c150633fd7ac6e0", + "verdict_sha256": "17845b6db0826010f994abc3e4cbb015977cff448f1083d33d4ea1826840d741" + }, + { + "case_id": "us__scenario_029__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "75136bd1-328f-47f4-b29d-e99a779b9fa0", + "prompt_sha256": "425282aaef2058fd92b24d6002a29dd7ee40cf66f7fbcfa0f2449979b42fca15", + "verdict_sha256": "13274b58aa487ed686c5539598e2733f0412f27a4b45bda02556ecafb8f3af82" + }, + { + "case_id": "us__scenario_029__state_refundable_credits", + "login": "pb-judge", + "session_id": "af7587ff-4d59-4674-992c-7f726a37a134", + "prompt_sha256": "5fac6b0cd57c94d3a3685aa5222303a3c1475de07f1885173b0d6e1eb8980503", + "verdict_sha256": "2cc0760d15f7d536a52b5a75ddcb0c5188e4188f56b185b9dc29464c88de1326" + }, + { + "case_id": "us__scenario_030__snap", + "login": "claude-18", + "session_id": "4029c89a-4b3c-4ccb-b929-bd26bc6cc62b", + "prompt_sha256": "3f7597cccb44f2494c1af740b867b917ed240380e7c9c7c9d2d3103141126871", + "verdict_sha256": "8eb4fa2cba889e3d8b0516646fc53a581af94b342f06ba49e6240e099bdbc18b" + }, + { + "case_id": "us__scenario_031__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "f8e66030-67c0-4d52-acde-7a066ec15187", + "prompt_sha256": "038f4a0ade9f0f1d3b7f3d693720d9338d536fd33ac9548a0b549cc12cbbd11d", + "verdict_sha256": "50e93ea238f2cda312559468aeb3e80267e308733c838a4d31c09903cc4602e3" + }, + { + "case_id": "us__scenario_032__child1_head_start_eligible", + "login": "pb-judge", + "session_id": "b8d6257c-4558-4da6-a36a-8ff6e94d22d9", + "prompt_sha256": "d0693813aa4587fee63f6a5f0a86ccd7c7c9896ba67e720ac179503a7b85d3ab", + "verdict_sha256": "0ce86babafa6a95898b1bd421b6192323efc8b875af4894401eb31eed30c2a6c" + }, + { + "case_id": "us__scenario_032__federal_refundable_credits", + "login": "pb-judge", + "session_id": "4c91a1c5-826b-413d-b9be-a8a8473c0cef", + "prompt_sha256": "c692ff7cb52bd2df79dd75099a3966dbb46a12a7b3ad9684d10d6d8d708f5bbd", + "verdict_sha256": "88cd884ba421dace2249090a7948cf1a729696a2c912f1521cf6adb222a362b5" + }, + { + "case_id": "us__scenario_032__payroll_tax", + "login": "pb-judge", + "session_id": "e28c5b5a-b336-4319-9471-14db61ccaff4", + "prompt_sha256": "14a645942f03a815209b9534763fa10822c957e1b2619dbee41c26ca6abc7844", + "verdict_sha256": "bf538655ee75fbab32127c3d2edcc64b8ddfc3be81c00e919afb5b2a1e8e01bb" + }, + { + "case_id": "us__scenario_032__state_refundable_credits", + "login": "pb-judge", + "session_id": "fbfd4b2b-6c73-4857-9eb8-608da161d05a", + "prompt_sha256": "858e34402d4b4b473d7843308f4b2fddfacc8d97c853032cd14a4bd4570908c1", + "verdict_sha256": "a73d2e58cde17a3428ec253f40e7c3422502a8b20a5edf4975cd97b653a399db" + }, + { + "case_id": "us__scenario_033__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "57144316-7653-41ed-b73e-68f97550746a", + "prompt_sha256": "d5dcf9d78866c9b78b87b9e38c76cc41476ecaa1e2479b9eedff46e2fee0ea75", + "verdict_sha256": "6337684e8c8766368358f1f37b49ba71a0915cf10b9b3183b60bf5882fc8be0c" + }, + { + "case_id": "us__scenario_033__payroll_tax", + "login": "pb-judge", + "session_id": "78b724fe-9225-4b3b-9e90-c090a3163bbb", + "prompt_sha256": "c9aa4c2d6fcb1c429d724297793a2a5fb209b50cca76c22b9f5d7bc408cda0bd", + "verdict_sha256": "5598d0c0d0f73374a2a854fa23c96e58fc36ab3b6c6d1133c3d5028ff15b50ef" + }, + { + "case_id": "us__scenario_036__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "ed360ebe-af47-470f-9ebf-e563cbdd2d6f", + "prompt_sha256": "12e2f8b805dc8403d6f527af6b8218f4de726ede07e1becfbcde2b7d77d008ae", + "verdict_sha256": "c635aecbf7b10f8d6acd2d83abc4b6db93d76c9bdbe55f6b8c19a3be245eac83" + }, + { + "case_id": "us__scenario_036__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "4d465247-7465-4399-89a2-0e47c1ffaf00", + "prompt_sha256": "0111a30f4716105d3be653c00a765631f7900881277474802d9a9d14d537bfc7", + "verdict_sha256": "e115b451376863639eee6497ef7c9482fb218c22db38315d72670046f2779ff7" + }, + { + "case_id": "us__scenario_037__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a69a7bf3-6d22-4d34-86a8-ef279ce0d485", + "prompt_sha256": "34b95bae28bcc8829cd1cc183210c49c2076006722dc47a774f83deb78ff5dfd", + "verdict_sha256": "65172a29c6d7e5dbc7765632afa4fefe6ec5d5d3a8191c6be8d2b8f53768d9aa" + }, + { + "case_id": "us__scenario_037__payroll_tax", + "login": "pb-judge", + "session_id": "e456e549-5829-4df6-b834-b1401d5b3e78", + "prompt_sha256": "794845071e5569220fe34637c8d32b60e72a547a755adcfab63277f96dc1ec2e", + "verdict_sha256": "06792bb759b9e082788825aab174be85f3a4ad49d16a83a2fbd915e52ffa3bd0" + }, + { + "case_id": "us__scenario_037__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "b606d140-0bf5-486b-a0d8-f171e3e8e5dc", + "prompt_sha256": "f8a5aeb255a33c077e71cc735833b1944ae1d4a7b5d95d645fb95d9294c2d208", + "verdict_sha256": "54b9e7cc8feb488238c99d13b0bdce37be112e9dc9c5520b11caca1b8d40abdf" + }, + { + "case_id": "us__scenario_038__child1_chip_eligible", + "login": "pb-judge", + "session_id": "99063547-cdb4-4082-aab7-a71bf834b5e4", + "prompt_sha256": "e1925f16f7e0f00d7d6ce5df76e316cd6132c22d9d6a72190cfe69688211dc89", + "verdict_sha256": "ca207c7a8003b639fdaef9d7f5a9544995e698fa44281e84af7e2155cc22e2bc" + }, + { + "case_id": "us__scenario_038__child2_chip_eligible", + "login": "pb-judge", + "session_id": "829156fa-b06f-4ec5-9e3a-a54b7fdfaced", + "prompt_sha256": "19b2f317f8aae78a6082166170b20059be8a377b88d9a20c410e0383fe398b5a", + "verdict_sha256": "b1ece33f65e3c872c1daf8d39d864fdf9d3c312f9d45b26da86449cf10aea184" + }, + { + "case_id": "us__scenario_038__federal_refundable_credits", + "login": "pb-judge", + "session_id": "2db224b2-a8dd-4825-b332-e18558e4e36a", + "prompt_sha256": "8440a3f49bb7ef5ffb70c82756e36a5f07b7e606212e8634961e49393e5eb402", + "verdict_sha256": "efae871ea4254767e578148d8f31d42afad5749c5bec08d045c99a18f5947367" + }, + { + "case_id": "us__scenario_038__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "e3c9c660-dd46-41f0-adb7-72bb1a1b8c6c", + "prompt_sha256": "04b06b5787ccbd8b67b38d9731e78bb56302bd7659c0332d8b8d754ac8a0d4f8", + "verdict_sha256": "0f9133393401e180cb8ed18309472ca6075dd20f575f006a54f89b7e6aa247df" + }, + { + "case_id": "us__scenario_038__self_employment_tax", + "login": "pb-judge", + "session_id": "b838c534-951c-47b8-a6ff-e15a9c060e53", + "prompt_sha256": "e0effbfa9c71fffc7b672bf908206adea97d94bb4ae8636efc1e22ab3f6d4e73", + "verdict_sha256": "272e45fa5dd4044c8c105defea5f4521461d6af8199eb9105d8c1810d3a82bc8" + }, + { + "case_id": "us__scenario_038__snap", + "login": "pb-judge", + "session_id": "dd3a383c-34a3-4ff1-a2b2-a6895d91d3fe", + "prompt_sha256": "827794e6727ec6439ac49d346cd4e6090ccea7c1ac15349062223f2befb553af", + "verdict_sha256": "baf8d558384a62ff9fa5fde71a1da0a7418d972391aa7b2a8abd70f210d1ab13" + }, + { + "case_id": "us__scenario_038__state_refundable_credits", + "login": "pb-judge", + "session_id": "5f7562e8-cb92-42b6-ab43-7c49e060c994", + "prompt_sha256": "09e72b96f1ba4e12a2a6b605e26ffa6662f6bcb4034f3f72ee80d6731773e2eb", + "verdict_sha256": "9785a81a921ea902039cfd3f1facec4f0b1ebeb4b84accac56b97c95da5b7df7" + }, + { + "case_id": "us__scenario_039__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "c37a6505-ddfb-47ec-97a3-1275ec742e08", + "prompt_sha256": "91056adf52e6a7fbc994dd1d0f0f9ca0db9c2d9d0fcf11b05b365d93205df487", + "verdict_sha256": "eccc9b71762cf4c9edfc4cd7c618b5dd0f544d7d4c7a8a69e9a9799b133cc860" + }, + { + "case_id": "us__scenario_039__head_medicare_eligible", + "login": "pb-judge", + "session_id": "c2988dd3-3a82-4936-b5f3-82c86feba030", + "prompt_sha256": "ebb802b330c78600500d62e4a6d7d1ba0d56323c5dba5c52a80f076f341d9a20", + "verdict_sha256": "a06d875a4465d0eef26e9c1cc1f8c31411c0213f3ac4cf16b0488f95314fb1dd" + }, + { + "case_id": "us__scenario_039__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "0d871cda-b44f-4730-bdca-171fa435dc81", + "prompt_sha256": "2c1730aa22455590cf03526bfbed9a6a0c2a33f2cc903dd5205adc5761f6ef62", + "verdict_sha256": "0cd1b0396319c8b5a12847afc235374257820dc3047bc1d043db973483b5c867" + }, + { + "case_id": "us__scenario_040__payroll_tax", + "login": "pb-judge", + "session_id": "013b2309-3d6e-481e-8bca-2b93a7a7ca8f", + "prompt_sha256": "1fed60f4dbe4e65da843db4b0bdb738697d29ee8a839e758bf0746b12b05f140", + "verdict_sha256": "13093a99dfbc9ff0c9edcdaca52ecd6e74ee88e7f2472e1c65a45bb1c2d46775" + }, + { + "case_id": "us__scenario_040__spouse_medicaid_eligible", + "login": "pb-judge", + "session_id": "b62317c8-2c58-437c-9c50-54af405dcfe3", + "prompt_sha256": "679d096fcf26e8174f6ceb7414d5a547aa6ab2dd2c9bc7c3612c3654b2573417", + "verdict_sha256": "0ab3d78dc2fff266c44f24738f400dd76fd110b5c2de5be0b49465d435a1f888" + }, + { + "case_id": "us__scenario_042__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "534770f4-fc43-40c9-805d-e0ead10e4781", + "prompt_sha256": "ad7c9420ce79d3676f6fae0a72c6b2cf85afcb5ee938973619011178994da199", + "verdict_sha256": "780085ada860c36c43125ac03a11dfd43557032cdf8c505d0b4f3c705e5e9676" + }, + { + "case_id": "us__scenario_042__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "fe091d6e-acdd-4e82-88db-255d4742dcd9", + "prompt_sha256": "6260c9b25f7b4c0cd3996ce2b0c40b7bd8dc33fcc627b51715bbdab7e2e9a9c9", + "verdict_sha256": "68eed26f001c269c63b9a788f584a673bc9b70328a0052f3e62dc477df55e8f4" + }, + { + "case_id": "us__scenario_043__payroll_tax", + "login": "pb-judge", + "session_id": "83782b94-a22d-401c-82ff-ed275ac47d97", + "prompt_sha256": "20880542a485c03ab864cabc8c6f911edbec932509d46f3f0d3576a877ded70d", + "verdict_sha256": "7a0dda0e9550e268300a5d61435749293eb7e02a7292b1656c7cb28b0efa7340" + }, + { + "case_id": "us__scenario_043__snap", + "login": "pb-judge", + "session_id": "fe72d5e7-caf1-4c46-bf64-0a56d66e82a7", + "prompt_sha256": "3a7435cd12a0c06608f916f999f18caa38b75193e97cd28f540bf5e1e33ffdd9", + "verdict_sha256": "adc8d2a7f27c6c5fb773f5beb555d14dbad7bf7c86a8befdb37ca024ae74ade5" + }, + { + "case_id": "us__scenario_043__state_refundable_credits", + "login": "claude-18", + "session_id": "cb28320f-258a-4258-8d34-13e3eb356d4e", + "prompt_sha256": "5c453100bf56dbb03a7662fb321ddfbf2ceaaf4e1b1d12b77840c5ea3731774d", + "verdict_sha256": "e06651bfa881e21214841a6e66a01ab869c72e2edf245d51613e28ed9a236996" + }, + { + "case_id": "us__scenario_045__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "48b522b7-ee7b-4373-86a0-2ca2f37f7c66", + "prompt_sha256": "c9a335e6710514f6914b76f0a18f9ef0852dd5e60175516bdfb1141219761950", + "verdict_sha256": "c42e06789c733ac8c469875e79ab4e01bd326c709157db02f50730aa0b9fc638" + }, + { + "case_id": "us__scenario_045__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "64fc7572-f700-4a06-a7b0-4db90da1d02e", + "prompt_sha256": "fb4ddd500fa197f8d137e369dca30f866ff6c39e00c698bcf809b99b061c97bb", + "verdict_sha256": "5930b0ab8846d8083ea8d718bf0cf016b880b762c6c205a70653611fe1b85281" + }, + { + "case_id": "us__scenario_045__state_refundable_credits", + "login": "pb-judge", + "session_id": "39fdf2f1-44ab-46ac-8467-f8984f1fa207", + "prompt_sha256": "864ef2fc785cc521ed3c07310a60b1c161f332857226f3ff6397ac4a19ce5ebd", + "verdict_sha256": "54557967511412c9833f205ef8754fce4f3942b65f12d7b059f50ea213d273ef" + }, + { + "case_id": "us__scenario_046__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "3eb765f5-b07e-40bd-980a-52a68139c70e", + "prompt_sha256": "5fd995e4e5bc0cc2a5e5e97b90769f4a24f31bc1638a83a46103bf547022d49d", + "verdict_sha256": "a197f268d2359139996d43ac45b84c416cdfae7188c4d3e1cb82901800f32404" + }, + { + "case_id": "us__scenario_046__payroll_tax", + "login": "pb-judge", + "session_id": "858f977d-8bda-44c3-b6bc-1f90547a22fc", + "prompt_sha256": "1e60d329f42a14c9c655c42f96c11ee11b7a2dc70d6f1f0fc6e34c8a7d2edef9", + "verdict_sha256": "fff22ccf33680808bd5beda05a5caae3e8548dd0d886a282539998f7009d7932" + }, + { + "case_id": "us__scenario_046__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "ced9eae5-56b4-4f12-bc5e-83766fe58671", + "prompt_sha256": "fb180d71e3e5468299518a204824aaae3f770c6eee51d01c85345811cd08333c", + "verdict_sha256": "5ab22390b0953a97f285d5f732abf4af983cfc4ce4f01b899e138c720988037b" + }, + { + "case_id": "us__scenario_048__payroll_tax", + "login": "pb-judge", + "session_id": "915771fa-3402-4a20-b851-ccde7f21e313", + "prompt_sha256": "99e4b79bbe41a7b315d325a46abc69ba9498be3a57979d75f54fcafed8eca589", + "verdict_sha256": "861d1399c9402c0eec443278d01c077f1a957b64ebc580b9831ad82513c21392" + }, + { + "case_id": "us__scenario_049__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "ed2da2af-69a6-4498-a601-bb91ac95a175", + "prompt_sha256": "d51e72b2182678f4cc5d2de3f4ff0b71982f17cbd6f932f9504a2686f0989838", + "verdict_sha256": "ea9eca35852e5dfc057fd18f4a48acda06731c4450a8d377cfb616a04366d351" + }, + { + "case_id": "us__scenario_049__payroll_tax", + "login": "pb-judge", + "session_id": "53b803df-a78d-449d-9894-75b62d505544", + "prompt_sha256": "f65338f628ea9b1f28e075fadc7f7ee7f8e3eed449268871cc1e11d4bfda9f63", + "verdict_sha256": "449d92ac7eaa937fe90b1a72d0b9eda9ee30dde9ad2c6f79c9ff840886d14bf5" + }, + { + "case_id": "us__scenario_051__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "35fe0e1b-57f0-47ab-bb0b-747554d95c1e", + "prompt_sha256": "339e57fde1bd4e33815c49964ea8e9128eab60eeb11fe52ba340e1ca0e92b8c0", + "verdict_sha256": "954245423973ab966a6925c305848d6d9e36059abca4d5b295d5ff4a724187e6" + }, + { + "case_id": "us__scenario_051__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "966c8a08-a7ba-4dfb-b737-28c2bd2ec371", + "prompt_sha256": "5ddf5d321a7d37b1950d3ce2a53e509d4015f0f31e5282c94b15f1ed0f714f37", + "verdict_sha256": "524f744c91fb09b511e995137a30465794edfd6c09e08517478057c5cff5ec50" + }, + { + "case_id": "us__scenario_052__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "d6f432fa-ee51-421a-95c4-4d05ce3ba5e8", + "prompt_sha256": "bda9d830697a13251decb90893854a0b1e4a0e5e3c9528784b6ee5dd43d21d79", + "verdict_sha256": "160f92579bdaa75197452ab92dc21c2e2c997933c55dc6ba6ac1f180f84d6da3" + }, + { + "case_id": "us__scenario_052__payroll_tax", + "login": "pb-judge", + "session_id": "bcd417e3-1005-46f9-83d1-622fb0a1cfe6", + "prompt_sha256": "6f9f6dfc1bf65ab3059a55e3c021bc97d612452d5ca41ba98f5edbc009fffca6", + "verdict_sha256": "ea24754abe4dc3a73d69bef03ead75d9a392e52b0b2dd1514ecdd8cbaf7b8b81" + }, + { + "case_id": "us__scenario_053__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "9a225239-9494-4612-be4c-5128d6fa289b", + "prompt_sha256": "0aece15329f0f8128ef89056d347e6ee8cd8bf623de96b2784cac0ad9968f356", + "verdict_sha256": "3ec04ab9775d5495a4e0767dd4a05a6361c04cca7bf8aff2a5a31bc9ba5cde74" + }, + { + "case_id": "us__scenario_053__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "98d02aa2-3ad7-42c4-adba-77640f2dac35", + "prompt_sha256": "c2360ffc115d1055ce64ba77c8fe3ea64bb2c6de29119f14ce957ae91377aafc", + "verdict_sha256": "eb6b52636f2920eadab585cbb2b329dc0b8df836b94132d0dcd980ffb28b6f50" + }, + { + "case_id": "us__scenario_053__state_refundable_credits", + "login": "pb-judge", + "session_id": "761a730a-820b-427a-814b-eb43763d9da9", + "prompt_sha256": "c3cc8ddb05d966f9c7889b87b924a9f9cdde79851f5e1914ccc8094db4b3dc7b", + "verdict_sha256": "d192a2e31d79be9970d9a776680b3836de779248dc04ab410cf66cd6083d99ec" + }, + { + "case_id": "us__scenario_054__child2_head_start_eligible", + "login": "pb-judge", + "session_id": "20c77c96-fae7-40ae-9c70-92eb1365c9f6", + "prompt_sha256": "ac06a81aeba13974672e68ca435af710fc1c0bf6b03c7ee20bc66c926fa4a3aa", + "verdict_sha256": "98972d0394e90d91587be10c820ec84c586539ea91a9cc646704c3be31d65e1c" + }, + { + "case_id": "us__scenario_054__federal_refundable_credits", + "login": "pb-judge", + "session_id": "a0a76afe-f516-4ec8-b682-59cf4e3fa6c7", + "prompt_sha256": "c0953192213a189f537db5dd765b70034230d7f43e1d31fc120e73eb52a748fd", + "verdict_sha256": "48f6300e0533f8c0a1109f8b98e4a73cc78053cb8975e8bada469716e3f94f75" + }, + { + "case_id": "us__scenario_054__head_wic_eligible", + "login": "pb-judge", + "session_id": "316bf1a2-6669-4791-b520-250ce32458e0", + "prompt_sha256": "ab128dc4837e05ab6282f10cc2b58b3c4142b6731716fdef879fafbbb264ddbb", + "verdict_sha256": "581369a3a8cf717d35f01c271231185c5f72879d6da34d600c5ce575a1e95194" + }, + { + "case_id": "us__scenario_054__self_employment_tax", + "login": "pb-judge", + "session_id": "c253a189-4964-41d6-a83c-8c8d4e9fde15", + "prompt_sha256": "2ebfe07a5c6540ac0761fbd0ae254e528f4ba984275bc7840d7cf5f9466f8845", + "verdict_sha256": "57247bf738aa515db9adcd330f0be7c530a7b7ece60254beb76c1be21bb89b70" + }, + { + "case_id": "us__scenario_054__snap", + "login": "pb-judge", + "session_id": "54016b18-693a-4ec0-b895-a7ddcaa56cac", + "prompt_sha256": "2ef3e32448ba649acc9e4748d10181333812ee026b7e9bf8eaa8c5c7953430bf", + "verdict_sha256": "98155640e04e269c42ce6ea92c4bdeaae3430e5ff8db94ef88caff0edeb42ab5" + }, + { + "case_id": "us__scenario_055__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "998e1b6c-d79f-4f85-bc9d-4962b03499d0", + "prompt_sha256": "2e1e73cc4f77b3727cc5d0dd99f701195195c784335a81bff72f043740e311e6", + "verdict_sha256": "22e9f5771d046cd0fecacc3de084c6f1a27b710035cc118a55dbc7022218cd1e" + }, + { + "case_id": "us__scenario_056__snap", + "login": "claude-18", + "session_id": "c6ccdf7e-ccec-4271-a7a7-594b4e49879f", + "prompt_sha256": "0841f62a55550474e31cc34cefb104a78240b982833036705488962539dbcaeb", + "verdict_sha256": "ebff25226d847b208498e7808663d70f8fb2540aee3dee16158b53d6810e5dc2" + }, + { + "case_id": "us__scenario_056__state_refundable_credits", + "login": "claude-18", + "session_id": "de4dc5c2-abcf-43a6-8dec-b323e4514a84", + "prompt_sha256": "bb2a5468d91dd9af3d64aab5096f59e216e9d23c8919ff94516d1ebce046f544", + "verdict_sha256": "7f1d11b681e01758782d470ab3c39ce75ea753d0e1d4c31f393c61214f273426" + }, + { + "case_id": "us__scenario_057__federal_refundable_credits", + "login": "claude-18", + "session_id": "056f499a-68f4-4ad8-9254-6f73c57d66fd", + "prompt_sha256": "290a9cdb70ea9936a6a26ebdd4a25fcff76bc74b144f23b4a259f101436192b1", + "verdict_sha256": "1e4fc20e30413dcf9fe24966da417aac8e9ae0bc073264e5c2db8faa7f333ce8" + }, + { + "case_id": "us__scenario_057__head_medicaid_eligible", + "login": "claude-18", + "session_id": "7747b960-155f-4228-a830-af06155479ae", + "prompt_sha256": "b24704c65dbbbd869b75808f3d75e9d21828139588965b21f6714959545e7726", + "verdict_sha256": "69f7a36659fea087602df9eebcfda9b865b978b138e30f94f8eb55302fead365" + }, + { + "case_id": "us__scenario_057__snap", + "login": "claude-18", + "session_id": "d8c8a852-3f94-4b1c-b766-df2b055163f6", + "prompt_sha256": "2fe4ce665dfcae55ac68b713f956ff8693154b30ac4be90e20f06a273c274738", + "verdict_sha256": "fd1ddc4fdf6d6db672262811721c2125fbbb4c0e42868b2ef849ab25b9c4bfe7" + }, + { + "case_id": "us__scenario_057__ssi", + "login": "pb-judge", + "session_id": "e887bd59-f8e1-4dd9-b5cf-4aac20054577", + "prompt_sha256": "df01dd5a4b49a6faf2084f60412099caf41e8f4a9c0b806a1f72728538036847", + "verdict_sha256": "608e3fb3dd7fe97c903bf813803ecb4feaf0a1660ef7b64bac2c3679d4915355" + }, + { + "case_id": "us__scenario_057__state_refundable_credits", + "login": "pb-judge", + "session_id": "3a65b7a2-3ce0-40b4-973a-4c0bcdc43e39", + "prompt_sha256": "18c1faaef19c5611435805d4940339f87a17db36fc06661dc7a473487ac4dea6", + "verdict_sha256": "9402e362c02a61a1ebab399d78a4d54a1f0045f15e05191adc63a1d1cc58e056" + }, + { + "case_id": "us__scenario_059__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "9e293c49-9181-463e-8bea-bfe344682ad4", + "prompt_sha256": "7630823ad57b1254214e8f0ef70da1e8190b0788c40500fbd78f9cce3df57e22", + "verdict_sha256": "bee1ada5651b68a206fee7f72a538e960513ca33b6fac5b36943fdf8506c6472" + }, + { + "case_id": "us__scenario_064__dependent1_medicaid_eligible", + "login": "pb-judge", + "session_id": "fd989cc4-b953-4bd0-ad2b-7529d3b86dc3", + "prompt_sha256": "c085aaf95619705f66e3c49bb040b2524463cb7d4a6513d1b1328ecc183a905c", + "verdict_sha256": "72a5dbb23c331bdf620a5b42c79dd1bb66dea050961c9e5655fb3d68aa3c6dfe" + }, + { + "case_id": "us__scenario_064__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "d9aa9ab2-85c5-441c-b535-a6de86e8ddf4", + "prompt_sha256": "f6c40913c583a53a403540c8ef1a88bf37caac15612749479af8928364f38150", + "verdict_sha256": "0fb38f969bee9b4bf2758fc10bc89a32b680793d3ada65e895537222cd1887d8" + }, + { + "case_id": "us__scenario_064__payroll_tax", + "login": "pb-judge", + "session_id": "98ee9ae8-b28e-4c06-aba9-9b4f3e63e16c", + "prompt_sha256": "793f11fb2e4264c06c8bfc04b6481d95a8ca78c704dad17ad8cf3f064d713ec3", + "verdict_sha256": "95cc2f85c5b4783ede28e16a0339e1aa2814b71d6d5481d58540b59cb82c39bd" + }, + { + "case_id": "us__scenario_064__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "6b41a121-9e54-4092-a414-f79e2485d561", + "prompt_sha256": "543d9f6397bbcf2d5967b0061382e3d0484d007ad9806ce15f9f7a198b34db93", + "verdict_sha256": "512cb0c25fe525da266ef7444c5db3e55effe5e499a861eb99c620f97620e0bb" + }, + { + "case_id": "us__scenario_066__federal_refundable_credits", + "login": "pb-judge", + "session_id": "49d5e5d6-ce48-4bd9-becc-178e1636bbbf", + "prompt_sha256": "3488c464832e9b71f70f3b888b7d5674bb772929bbeef96b0784858f5e6e3ee7", + "verdict_sha256": "dab99af16eee28a7b865ef93c51301b5b1576455d5e8d6530131cce5a83f82f8" + }, + { + "case_id": "us__scenario_066__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "e9f37429-6b25-47a3-85b9-21b8ec0d1c1f", + "prompt_sha256": "6dab1279c40d43495309d9c28c93ec8f44cf7229792ce2d3b612c62453ead0a1", + "verdict_sha256": "9e1e171ff356566c0adc18804de7165dc84df48ca50bd25db558ba5962cc4e73" + }, + { + "case_id": "us__scenario_066__snap", + "login": "pb-judge", + "session_id": "d85debfe-2ade-4373-b6ac-344c8fe70853", + "prompt_sha256": "ff356e52dc68566f3a0791734a15571d5787e62c5e394d123a75718c6522ff07", + "verdict_sha256": "2344c626778ff071da05cce666d262d80cae0621b70e0915ee6cc41de04b75cb" + }, + { + "case_id": "us__scenario_066__state_refundable_credits", + "login": "pb-judge", + "session_id": "593aaa8a-385e-42df-a812-7c2eef93ba26", + "prompt_sha256": "9724860d24732c5128185dfe03387ef81d99af4ee501264ec04091df93985b21", + "verdict_sha256": "c4bf4bff594fbc65eb8cb19015bc9fee7b6a6e1de057c5d950bd11ef00c74f1b" + }, + { + "case_id": "us__scenario_067__dependent1_medicare_eligible", + "login": "pb-judge", + "session_id": "0d4d2768-ef03-40c9-b829-8e3fc045b1d4", + "prompt_sha256": "7cbb2316725e09d821ad990bb3a9b05e91b8ce3be2e9c3c51edf5fb18648e325", + "verdict_sha256": "734bb779211053f76dedeaa438166ff900e74be67e19e163850a6b3812d8880a" + }, + { + "case_id": "us__scenario_067__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "dbf0c5d1-1696-49f3-808e-6b39ec0ac0d7", + "prompt_sha256": "8c7b1db90642ddf68f21b300752931545fe72caba1a597757133f8499619aba2", + "verdict_sha256": "e8b41e5f42e6064c546bfc46e9abc1f0489939206bb03013d1b4bdfcf3f3c38b" + }, + { + "case_id": "us__scenario_067__local_income_tax", + "login": "claude-18", + "session_id": "5fbdc41b-5a02-4cce-9a7d-18f4893ef390", + "prompt_sha256": "5b470a6d46dbc5ebe963e5c358aa77d19ce104c608df400f7a85f28907457406", + "verdict_sha256": "14c23bf7f38ae94b31c2f01fda8774837e4b599dcd43f38e2d21025c3cf19531" + }, + { + "case_id": "us__scenario_067__payroll_tax", + "login": "pb-judge", + "session_id": "cd21db5d-102d-428f-bc00-e77918721f48", + "prompt_sha256": "47bf89f8663077a8d90929ab5ebe4c70b9f27a7c0a6e4055067d781e4d5fc5ff", + "verdict_sha256": "1349613eb90210edd684be71c1cf5d8149f64fcf09eb4e1b7f23633fde1133a2" + }, + { + "case_id": "us__scenario_067__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "ace847d3-48a8-4d93-be34-a0899c5f242e", + "prompt_sha256": "a7ce739f90dae7c9f0d3f81592ff384e8e022d2cdd74be15b6dd86fe4befbe01", + "verdict_sha256": "6de95aae28165e100da33a4cd7a31044b8bc3c4263cc48666d07b3fadcab5e3e" + }, + { + "case_id": "us__scenario_068__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "f6b51a1d-a61d-4089-85ed-625ea0d52bc5", + "prompt_sha256": "46b0d4e8322b8ec2d2d470abb7f0089633a3abb859349dbf91c75b7ef1bd53ef", + "verdict_sha256": "9dce4b467687663b989a4922ad1232ad280e549bfcebe168bd53de6beb369828" + }, + { + "case_id": "us__scenario_068__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a0cdcab4-0f8e-40eb-9a84-245a42d39b9c", + "prompt_sha256": "414748235f231fb9316129e0ae2b3d14ee2fe8f18a6e011e9537d14bbef7ceb1", + "verdict_sha256": "d7e22cc43ffdcb09f90fe0b94f51d6892460351d2151da1c4adad04afd88f3a4" + }, + { + "case_id": "us__scenario_070__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "fd3401cb-d525-4fa7-a4ca-d1d18f39430e", + "prompt_sha256": "c070792327ce4e149633446cdfa69a16c231401f9268befe9a6305e1af6f59fd", + "verdict_sha256": "5e8dbb962ffb599aa2400add459c1d9c3a1d888c9fa5de60eeea904a8bbda31d" + }, + { + "case_id": "us__scenario_070__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "e7d5246f-b549-4b93-b8fa-4cbb99ebb3e0", + "prompt_sha256": "c2fa1e225bb2cda9f138ac3a7640e519070907cf5dad0c9782e6b943f63ec164", + "verdict_sha256": "e21b0d38d07ba0dc368bf37f2aa6ce0cf3859df1cf9b095dcc5550b2c292b259" + }, + { + "case_id": "us__scenario_073__snap", + "login": "pb-judge", + "session_id": "25e27587-0e77-4cfb-be47-01147eee0676", + "prompt_sha256": "b4e5052fbb45e2022475ac12e736734229b00c784c57ebd2130decb8e9f69b9f", + "verdict_sha256": "56073985c932e4c369df4e016b5afb8626c1c6415f57255a055e8efee73d28f5" + }, + { + "case_id": "us__scenario_074__head_medicare_eligible", + "login": "pb-judge", + "session_id": "20b570bf-f968-4617-8e48-5fd9936246f2", + "prompt_sha256": "f0a99534476b8780cd94916f15dda490a55fd7c013917070cf3295d8b3e01d4a", + "verdict_sha256": "e21095233ff1c9c56ba96883ef30dc220c12b6a9082ab80b762fefff10308762" + }, + { + "case_id": "us__scenario_075__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "fd1c98ef-d9e8-425d-966e-fc65312ae4b4", + "prompt_sha256": "9c7bc63ca553f35e6623ed395e2b082752cabd103526192b681e44b5f0fd2a88", + "verdict_sha256": "62c7d4602bb2d32f6f3156c45915eeaf2da6be2f658a1772f2d0496f70e3cff0" + }, + { + "case_id": "us__scenario_075__payroll_tax", + "login": "pb-judge", + "session_id": "d3841917-2adc-4269-aae8-d8cfe80f4aa2", + "prompt_sha256": "7d1c1f7e262b9e6a0c67063f196ea4d86b385e66f4164bebd70d580ea769c936", + "verdict_sha256": "1429898520996c496b6ddf693ce517f323fccaa59dfaac79c35275a41ed74a9e" + }, + { + "case_id": "us__scenario_076__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "431dc77e-c676-473e-ba8a-e0f0ef4d948d", + "prompt_sha256": "dc3b89eae9ffc45291dbad75cc6a7a538623cbec700edadf9a1c8e23a6ba919f", + "verdict_sha256": "e83e13c40d5eaac576fd58938670de1bc1b9bb20eee203858de6a3ab219727a6" + }, + { + "case_id": "us__scenario_076__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "a941064e-fb76-4868-883a-4ea9502d2aba", + "prompt_sha256": "d37ad790c66a0ba9491f8f1e8a5bc247b854d6077cea89abf6ead32bdff41f22", + "verdict_sha256": "8e4680ae3a223014470cb19b74e864522c20ae9e4a3217b126068bb5a868960e" + }, + { + "case_id": "us__scenario_076__state_refundable_credits", + "login": "pb-judge", + "session_id": "d5c0049c-1e03-478d-b3e4-0881914c66f3", + "prompt_sha256": "bcaf490a0941ac20f02f335d9b5d4f80176826dbf5cd68726a0b84ddbf296c3c", + "verdict_sha256": "b6f89c06b7b067b71609f65badb0f5362a037d32219c89ce0f18cef73fa79534" + }, + { + "case_id": "us__scenario_077__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "297244d0-71f8-4eee-8134-b304bacd07ae", + "prompt_sha256": "cbc6469c3f879b5059834f1fdce059167be41e86e69ba58a79345011238e7c65", + "verdict_sha256": "19bcb97950e18f6ce00439f3fb7f9ca0530fc7fea0ad55e506c9edd9580f4fa2" + }, + { + "case_id": "us__scenario_077__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "11146d4e-cafd-4d14-8e50-79c57ecd714b", + "prompt_sha256": "dc10c82a8bd5cc2bba4ac13701fd6cf52aa8f4ccbed4f72d7ea6b67e71c5efbf", + "verdict_sha256": "ac25f9a957e6df1a804ffbc0826fb5f27581670aedfe4512f1b136e2654f5e6f" + }, + { + "case_id": "us__scenario_078__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a524dbfb-deb5-4666-b497-1a693c4740a4", + "prompt_sha256": "d0462157251cae830add57246b1b10cd7ade4bfce2398779cf587a7aafc3a79e", + "verdict_sha256": "b2d91b793cbe19150b176bbeb6204a4b7809879f36bdfc1042a02906f84bff97" + }, + { + "case_id": "us__scenario_078__payroll_tax", + "login": "pb-judge", + "session_id": "5277fd90-2219-42ec-9c7c-7734804b9b95", + "prompt_sha256": "bbff94fca330f1b1a09f6078971c8e7827ecc23960ea0bc7a876e8a9d165f7d5", + "verdict_sha256": "2c3b31657284cf46b52990d81c1ee7cf26f4c535ad1f714e80210a7952bb23c5" + }, + { + "case_id": "us__scenario_078__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "5c6dec08-5519-4447-a020-8adc01cd6d7b", + "prompt_sha256": "bba3aa10ce12471805a4c87dda26a41b9bf337a5565ca2ff88c4e660d7519cb3", + "verdict_sha256": "2fb4cbdb4f167b71c15c97d78f0408a783aa69dff95c8f8dd78b170b850d6dcb" + }, + { + "case_id": "us__scenario_079__snap", + "login": "pb-judge", + "session_id": "84ebba5d-3f63-4657-9702-978203416bc1", + "prompt_sha256": "358b7d066755e939451e115bd684051e1dee80621ce4855a44e8070dde7c2246", + "verdict_sha256": "5d720888ea025cbb7a906771e21385bd1dae1bc787b4fc1ef7ea92b2a336ad43" + }, + { + "case_id": "us__scenario_079__ssi", + "login": "pb-judge", + "session_id": "4437775c-fa86-415d-9bf7-86d2562f3bb1", + "prompt_sha256": "933e09f2a7ad4721d29414b4472d27d7e77e7d0f3c80936477f5d2c93156d043", + "verdict_sha256": "13431be4f0c9bc6fef0ce3f18db21bcc0fda87aaf5e500f44f26573fab6930e3" + }, + { + "case_id": "us__scenario_079__state_refundable_credits", + "login": "pb-judge", + "session_id": "77c6b7f3-2cce-4f09-a344-d59eab422c00", + "prompt_sha256": "af85643d6d3badc459565b10d78abd1bf9dc3c31b61196c9dcf6710924995945", + "verdict_sha256": "ccdc6b959d15ad34e6781e8d45b623ec61eaee88f9194bf000b6e4be3f6f9739" + }, + { + "case_id": "us__scenario_080__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "05f34057-d48a-4143-b424-9d90613f934e", + "prompt_sha256": "8db9ea4765f350366b1609d0ec8a322bc46e106eaa2ea8df277872bdf357bb09", + "verdict_sha256": "89e224563d973dcea92e8a030b2ae805db18454dc8d918115005a5e3af0449b6" + }, + { + "case_id": "us__scenario_080__snap", + "login": "pb-judge", + "session_id": "28284bd5-509a-4531-a584-2b765d980dee", + "prompt_sha256": "6f1f87267f9c7160cd0cf2e19417e42be61db743242b5f3ac6b4457378fc09db", + "verdict_sha256": "00461d72bec0c63e44f74e1119a5a051bd8db0d509dddc95aded9136c2c92cf1" + }, + { + "case_id": "us__scenario_081__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "11781fe5-7294-4a30-888c-fd35446b8403", + "prompt_sha256": "01f58ab354674e5edeacc02fda7ab4c7bd8992218e5223001c288b8a1f24791d", + "verdict_sha256": "11819fede6b291f15a7feb97469c3975b1ebd32c53cf3d11a697df64deae28c7" + }, + { + "case_id": "us__scenario_081__payroll_tax", + "login": "pb-judge", + "session_id": "0d2e7f80-7d88-4fa3-8850-9d60762bc9de", + "prompt_sha256": "46c6f5fadadab8ee0a97416418cbce9def84fcfb84513efda1a389a2b9062492", + "verdict_sha256": "ef2fc03439ff51bc446cee92fabe93e28ebd5e988d31d2173341a7ec0fb8c1d4" + }, + { + "case_id": "us__scenario_081__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "e9da4045-3729-48c9-b03d-1b7d41625989", + "prompt_sha256": "6a2d1758f67f045d081aee8b67c8f223fc6b95b94fa10ad6fa3b48cb308d6857", + "verdict_sha256": "434ef3ffad3c4c9e92e1d1a282fd3a08018f1d7f5cf2987227bb60b86a733995" + }, + { + "case_id": "us__scenario_082__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "f91d8255-3416-4c28-becf-c822204ad66d", + "prompt_sha256": "e2cdaab17ab3730ec11c42b32e066e2f21c5b0a9bb0351cf7eb7bde3e3bc2fc9", + "verdict_sha256": "2b3aa05db39d4f909d50328dbccc8de00a5d42e4ffab09fb2ef9f31b2608ce72" + }, + { + "case_id": "us__scenario_082__local_income_tax", + "login": "pb-judge", + "session_id": "556d6e27-b7fd-4298-a929-78e62cbc3dec", + "prompt_sha256": "7e5e56f143679d05ce5e6a85312929eb7775619d74092b959eea6c0c5ce60562", + "verdict_sha256": "12dbd03b078e5536dfd64d9a2361d3da5bd6103062e9fb62b9464eee5a48ae0b" + }, + { + "case_id": "us__scenario_082__payroll_tax", + "login": "pb-judge", + "session_id": "035941e8-cf49-476f-80a7-c1254b2dd2d7", + "prompt_sha256": "748242baa478d1d1c9209b9c4e91d29089b5f17b93f7fb49a25ca783d8bb5100", + "verdict_sha256": "75e0dc06d14f919cdee0a2e82a8611ea2babb73c2d62d4dbe1d507ccf9e5edf7" + }, + { + "case_id": "us__scenario_082__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "a26034f0-7058-4526-9911-1aab9bd1b3e8", + "prompt_sha256": "b35e2afeac00102faf9d716554e4c800a8ee4dc2e3b7d0c6feb2d705b51ad7b2", + "verdict_sha256": "248bde3027955427bc3b54651f88b86d447a2f748731ff0a17ca1e3a70479bca" + }, + { + "case_id": "us__scenario_082__state_refundable_credits", + "login": "claude-18", + "session_id": "9e14eb57-4dce-4668-a594-3206473efcff", + "prompt_sha256": "106165a155117af10eacca2a00b893d8f4e7a165afc92e7445560f2aa81b0bac", + "verdict_sha256": "48ae58d8e85ab2ea0ebabdfa1bab08f5d8e7cf4ffb2a8fb0dbfc8b42bed2e14d" + }, + { + "case_id": "us__scenario_083__payroll_tax", + "login": "pb-judge", + "session_id": "bf6a79df-3ef5-47c7-b27c-078d188d09a0", + "prompt_sha256": "c4391157a1e537f41f75ed7fd2cda7074fe1572909fcd9100b287dbc18f0eca2", + "verdict_sha256": "a1ca26d419510251a0ce99bfd8af3bd41c00f754a7528742228d27c0b4bb2eaf" + }, + { + "case_id": "us__scenario_084__head_medicare_eligible", + "login": "pb-judge", + "session_id": "562d9be7-1858-4187-94ca-5fe0357dcd80", + "prompt_sha256": "8197979f49d7305ad0ff467b6d11d27ccd828c3d2d098bf90d0e376a4eae9e83", + "verdict_sha256": "8a3f550550750e55f2faf3c701a271d21512800edff4229255fd002cb710362c" + }, + { + "case_id": "us__scenario_085__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "471eb5b6-e3cd-47d2-93c3-c498c64a978f", + "prompt_sha256": "a273818e33f8e7f7f84955fe2cc32f5326d7c8967ae0818d79f5486b96eaa222", + "verdict_sha256": "ec50a0771a8e881f257d2eefcf709894a42ce36c7804ab120237b35e3dc7f7c8" + }, + { + "case_id": "us__scenario_086__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "e8c0e54f-6911-4137-aa62-1f53668d6f40", + "prompt_sha256": "9272ab85afceb414fc2f8c418874532f6d91b36a6570bb5231ed3e3dd7f3b993", + "verdict_sha256": "225f0748e81e9e4434bea454fdb99a5b496a67fd9504f477508623759ea52cac" + }, + { + "case_id": "us__scenario_086__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "28cdf892-9f1e-46d8-9978-bba46f40ce35", + "prompt_sha256": "5f8f2ae329eaae2811d349cb832db188a7c721ce9a507f2b9d213194b9b9d82b", + "verdict_sha256": "bb0c19cbd2b1e7d51dc3eba71e3958d7cfa527bd3bb652811738d50c1c522ed5" + }, + { + "case_id": "us__scenario_089__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a8dcc848-5162-4980-b1d3-fc37b8238e5d", + "prompt_sha256": "97bc7070d9808eebcab4580f0b8d2320fce32585479156eb233be13a590127c1", + "verdict_sha256": "74144742eaf21fb93146958ad63f28eac70c7334dfef1fcb41a826d8d349c794" + }, + { + "case_id": "us__scenario_089__payroll_tax", + "login": "pb-judge", + "session_id": "846633ba-b4c1-473f-83b0-39ec43cb119d", + "prompt_sha256": "3f665497df67834b1bf33fe1ad8ba3bd0b8d2d2ced08e014dcc7d4296efc4359", + "verdict_sha256": "f891b02dc2c95d4981e136811d6076c9c4a0a919ae3288364dc32a39635c9679" + }, + { + "case_id": "us__scenario_089__self_employment_tax", + "login": "pb-judge", + "session_id": "1146e62f-5573-4841-99bb-326c7fc68da3", + "prompt_sha256": "31b4068a5c1b9409fc907ce06fdba962d424a2b20fb8f3c88bb21d7053b5cc3b", + "verdict_sha256": "db7bf9ff52dde77a27ef2490a502d241c983fb3f8662535af91719a181ccaa38" + }, + { + "case_id": "us__scenario_089__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "255d7a35-6ee9-4656-ada4-2ec488f0395c", + "prompt_sha256": "966d4e9be835310eac25f9dc755cbe44e9329f5e6be6d2adcdf3ac8609ddfcfb", + "verdict_sha256": "819d83f792344de3bf9d870cd7d738d9129575919ba75ac679bd2a870f8e81c7" + }, + { + "case_id": "us__scenario_091__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "aea18b6a-022c-4376-8fe5-4a18c3e979c2", + "prompt_sha256": "25abe3dab1c432cf24370f63e280cd751621416abd44eeb48568e7a375503556", + "verdict_sha256": "65b18a70ecd45728ca9402a7b6cc7e4b4668f3b3c8f5fcbd04c038ec4fb97d0e" + }, + { + "case_id": "us__scenario_091__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "c657b761-945a-457a-b93b-4643ca2c00fa", + "prompt_sha256": "ac9cd8333e6a2cbddf3cfb389ca1f9265fe2c238ad311954200e8db16abbf90e", + "verdict_sha256": "b6e8b55b903170f86e341c8a3aee723b1d174d78659fb811d979c404dfd65854" + }, + { + "case_id": "us__scenario_093__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "f7f67f52-61bd-4e43-b2ca-569cb9711c83", + "prompt_sha256": "2e090f4f4dcda49240d4547494268853936b62f8dde3dfdc2ccf4238633ffae5", + "verdict_sha256": "64c3a1b7c1e9dba12bd5460cf18033a610f4f11ed3c685a1977986ef3342c086" + }, + { + "case_id": "us__scenario_093__payroll_tax", + "login": "pb-judge", + "session_id": "ce149e11-612e-4c44-a6f0-50abc3003836", + "prompt_sha256": "38652b10eb814185590450f18abe2a4f99e8f38cfd8834c0fac5f92ea6c4261c", + "verdict_sha256": "90d5369aeb98e04288cb4edc99f3d86afc3432000627f5ee2fc3f95138dd95c0" + }, + { + "case_id": "us__scenario_093__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "1074ed6b-b366-464c-beec-f776c3a44f40", + "prompt_sha256": "1abe54eb43624689045698b8386a96cfde175c1d2e3a95d3e886a1fded6e1621", + "verdict_sha256": "3bb92b597392168d3797344d6d533f5469324efe8f9bbcc246eed79e132a000e" + }, + { + "case_id": "us__scenario_095__self_employment_tax", + "login": "pb-judge", + "session_id": "d94d5472-9c9b-4635-a2bd-66069c47e49b", + "prompt_sha256": "3c8a6a24a49d2fc4d33018793b727659658fe8a35cfd1a517ce44fdf107b8d4b", + "verdict_sha256": "38f626745c3acd474cf9d1892074757a174dcdef67640ee5d9f66b152842ffda" + }, + { + "case_id": "us__scenario_099__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "e212b65d-b2c3-4ef4-8bfd-342afecdbb02", + "prompt_sha256": "fb80ff34599d7414e348de256c9f403218b26edaba12015be17d764626f203e7", + "verdict_sha256": "18524df596840bae585909732f4096e48a30a6d36f506916cf59cb8a3bf3da89" + }, + { + "case_id": "us__scenario_099__free_school_meals_eligible", + "login": "pb-judge", + "session_id": "a450bef3-43f5-490f-9980-a5e0d97688ce", + "prompt_sha256": "d7d1fdf83cad78a8e90e7d354f0f22b2bd3c1866d329379ab5d1ad8e079a199a", + "verdict_sha256": "84af914514a8e4bdeea05bbaec708f44440faa2a6975997dd5d3c4c1d7ea51f3" + }, + { + "case_id": "us__scenario_099__payroll_tax", + "login": "pb-judge", + "session_id": "dbb9ea45-7beb-4293-bc51-848387e96e9c", + "prompt_sha256": "fb1328b0a9b09f1a008d02534a2df6ab58e48d1734efb18dcdf6e138e45726db", + "verdict_sha256": "2256c2ad86cee224488587f5d01b2f5664a5cd550e683e180d2ed3e5f8e8b01d" + }, + { + "case_id": "us__scenario_099__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "68bac3e6-5189-43a4-b5ac-e825a73ee743", + "prompt_sha256": "4e70fc18edd1d27e2ef589ae02325425bf0a8115221edbad4fa94bcaa38dc571", + "verdict_sha256": "5a96439af716b7d43516665a1bdcc7db7d9b3f72cda484764c67f88daa996314" + }, + { + "case_id": "us__scenario_100__child1_head_start_eligible", + "login": "pb-judge", + "session_id": "f8471f19-84ab-4fa1-aa7e-fa4978977286", + "prompt_sha256": "49c5267e1955e218824edb896ef6afe8513c15900fcb3fa8162d8464e1d1dbec", + "verdict_sha256": "a1bc7cfcb12a742bb47862d3d6a680916ba4dd02badedb72075603aea79cbbd6" + }, + { + "case_id": "us__scenario_100__child2_wic_eligible", + "login": "pb-judge", + "session_id": "6b885259-b313-4e38-80f1-23245c799c17", + "prompt_sha256": "34755e3b9718b10316b26a50b9417140686100682ea3f752cb3c65eb234f96e0", + "verdict_sha256": "163409e6e0817d44d1e1be2755c836b6154e25357a96be94c065a16cfd557b96" + }, + { + "case_id": "us__scenario_100__federal_refundable_credits", + "login": "pb-judge", + "session_id": "8cf25a7e-a95a-4e63-8e03-e6a3a7222877", + "prompt_sha256": "dedc78aa47bf294cc1f15018fe354974a0d22a67d0997d1882ddb3cf3c1868dd", + "verdict_sha256": "a9babaf7cdac3da5f128f36705badf7b3efbaff1f8e94ee9e88886406c5421fa" + }, + { + "case_id": "us__scenario_100__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "ef32de82-45e8-4bf9-991d-8f88f82379a3", + "prompt_sha256": "48e03b27b24ed322a410324bbd575170ecdc529b7ac7a62452345654b61918c5", + "verdict_sha256": "b517187637972fb7d544a8f1f29773907108f9f95dd54d08474954e9d2bb9b02" + }, + { + "case_id": "us__scenario_100__snap", + "login": "pb-judge", + "session_id": "521a2c88-c48f-45e9-afd8-b5b1ce2d2ca8", + "prompt_sha256": "5546ec301eba6b35264477f4f5cbe928a1fa521b5ee175a8e066e8b354731893", + "verdict_sha256": "74f15fab7d138af0040de665895e5ef17bdd031bad67b387495112ceffbfdf55" + }, + { + "case_id": "us__scenario_100__state_refundable_credits", + "login": "pb-judge", + "session_id": "27f4e8c8-4363-40f6-9fbe-d48a7d6d05e4", + "prompt_sha256": "73a92e3143934b51d7fc21569bae2d19e4cd1ff6cd3b9fde63771b666ad05de4", + "verdict_sha256": "19c2d6d31341942983ad670f4247af19ea823faaaadd7993cc622516b59abe2c" + }, + { + "case_id": "us__scenario_100__tanf", + "login": "pb-judge", + "session_id": "f016d6cd-7f65-47de-9583-d73ab89293e5", + "prompt_sha256": "c70fb95cf994c9ce31e0bdf5b07096f9b3f04666580b32c6ee2ad6eac2718036", + "verdict_sha256": "61106f7a525f1e786079128133e9b650038d4ed0b14f2371ec1a7f69cc35fd4a" + }, + { + "case_id": "us__scenario_101__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "c0944326-489e-45ff-bb18-4255f0b9828f", + "prompt_sha256": "ed69208c41533212218261eadd9127effbc19334becce883d0b24af3cdb87c09", + "verdict_sha256": "6ded4f3cf867cc12816d956c9344febe03ca1b4aed9b2e0ea24918d6d26ad3e1" + }, + { + "case_id": "us__scenario_101__payroll_tax", + "login": "pb-judge", + "session_id": "edff64f8-25b4-4d50-b54c-d6c81fdaf9aa", + "prompt_sha256": "bed608f5de9caef1a933f0990d62a08367b3615af714ebf9aa0aa75b6e3f1fec", + "verdict_sha256": "99fdc0ed8e493e4a348c42dc035013c006a9c0d4463c25e46c7c3382b1d97e7d" + }, + { + "case_id": "us__scenario_102__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "3669fdee-a6f8-4be1-bc95-1ff3b1260cbd", + "prompt_sha256": "1f4857f021de2c2cc2a579256a00a01f633850229b58de0d9df7092c5fa2b928", + "verdict_sha256": "8c9715481ea35eab3a5881a01091de3540da5a1ea5f41de266a0fe8024b263e4" + }, + { + "case_id": "us__scenario_102__payroll_tax", + "login": "pb-judge", + "session_id": "5118f828-fcb3-4ebc-b3d5-3eeaa2cb7fd3", + "prompt_sha256": "9e82e5aad615feda16e19f80961d7cfa34d536d6115c6f5684366e8b8a984de5", + "verdict_sha256": "49094627773fcf98ec5b1c460d43f8bddc2f72e90d5dec41ec2daa4a583f3605" + }, + { + "case_id": "us__scenario_102__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "5a2079e7-6128-43c9-b55f-9d4033c2e5e5", + "prompt_sha256": "274d7a14d83b6264b2c437f0754c09843e9d2f6c834a67d5331077e1935eea38", + "verdict_sha256": "67b37ee4f21d701c7986e8bf3f51ba84db82ec6f1d782b2da8d22cc989b5439c" + }, + { + "case_id": "us__scenario_108__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "0e28d207-a162-470e-833a-19181bac49c7", + "prompt_sha256": "db5a96055dfef40d51d99ae837e5b5992280904d4fa813876ea34b4533686a91", + "verdict_sha256": "ac8f0f7044564eada7aa064f3a30c5e854fb06c21936ca03bd2b096986840fda" + }, + { + "case_id": "us__scenario_108__payroll_tax", + "login": "pb-judge", + "session_id": "94fa0756-b59f-4e31-ad0a-231e217593c9", + "prompt_sha256": "31501e3f7b6106c30f3e88cd461b81b1c25c0f2778a111b77734291864028f81", + "verdict_sha256": "07b5f9a7fd9978bfd4172f80262fee544dbb2b871f1cb1503bcef419707fd653" + }, + { + "case_id": "us__scenario_108__snap", + "login": "pb-judge", + "session_id": "47144cd5-68ac-4d27-bc14-0033711c2305", + "prompt_sha256": "b3f71e02d3c2ce99092eb763138308233e74ac28a9705b7fafe2f1b1534c0e38", + "verdict_sha256": "ff1af8ca8475b1bed5d17a3af7e079f2a0c18fe1424172b5586c47b920baae9e" + }, + { + "case_id": "us__scenario_108__state_refundable_credits", + "login": "pb-judge", + "session_id": "ddb37c11-44c6-4fe1-af10-0ba80cf6f91e", + "prompt_sha256": "b9519c04fdc693206d8d5778de6cc014edfb68d02afdd645679fb9ab8bce80e0", + "verdict_sha256": "52e48492b54957b7ff7b983bc959cb1472a5cdb46a5f115baa78000641a9f88f" + }, + { + "case_id": "us__scenario_109__federal_refundable_credits", + "login": "pb-judge", + "session_id": "a88530ff-9dee-4099-96af-f4dbb4ca2004", + "prompt_sha256": "acbdaf149b8a09bf55f5c20dfa7b12277165727c3023b7e8f620335d331c9e18", + "verdict_sha256": "02d325ae1ee91abd23a80ea484eced2a6f169ae98a10eaa328c6de768434fe4d" + }, + { + "case_id": "us__scenario_109__snap", + "login": "pb-judge", + "session_id": "1240d70b-ff46-4d42-978d-7c69744f5c13", + "prompt_sha256": "3e5c7fb6d09f3c8a811a6c153ae57b5245e907bfe126c17ee48929a688320bc7", + "verdict_sha256": "9b7a7dbd91b86c0989d9ca17060ed9858a1f75c6f62857245ff6c4cae51e4d0a" + }, + { + "case_id": "us__scenario_110__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "07eeef2e-26d5-4f31-9b9e-45a47017a97f", + "prompt_sha256": "68fd6bb9d44c315a4d869603a9ffd143e99b64430d2a4bb66db58868ddec819f", + "verdict_sha256": "b8e7ffd62f3b7a0ce5f9ecbcb59adbd9ac09f97502b40b47b86e41b094947765" + }, + { + "case_id": "us__scenario_110__payroll_tax", + "login": "pb-judge", + "session_id": "f02ba935-06b4-470b-bf86-8096f574bd0a", + "prompt_sha256": "abb5e050dded944da178dc2c39c85ad041fceccd4fa99f5374a9ce6d010f4f53", + "verdict_sha256": "991c2b3ed25a7714261e4df135f08e750fe84fb1af72cc83aa2849cb711f8969" + }, + { + "case_id": "us__scenario_110__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "11f91134-11c2-4cbd-9de4-57911d033a5d", + "prompt_sha256": "43aad489d0c1a53b8a030b85f0aa0485f9eb77cdf019ccef3e3e4843f135cd7c", + "verdict_sha256": "c0f68bed3abfb94d9eb9887ec8292a0b8a0cbeb761ea9841c8cab96a106ec9df" + }, + { + "case_id": "us__scenario_111__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "bb98a4df-bbcf-455c-bd83-111632a2b4e6", + "prompt_sha256": "a430153ce573528490e51ba0f40315658e0569a761c172d4fb55e336e007df14", + "verdict_sha256": "f1bcbd2a52d30f24f2da19e19df059578e4b017e943372f763c34343a48a5bad" + }, + { + "case_id": "us__scenario_111__spouse_medicaid_eligible", + "login": "pb-judge", + "session_id": "a9a3873f-ac10-45f6-9c5b-5cc47966e9be", + "prompt_sha256": "10eadd161726da6ef4678285641680ba6165cfb04ff1f3e87be611520f7ed144", + "verdict_sha256": "b99d3dcc48b45ae2fdd187409fc6601176da2db82907c6abd08a8228e4e5cf65" + }, + { + "case_id": "us__scenario_112__federal_refundable_credits", + "login": "pb-judge", + "session_id": "6ec1d548-b90f-4d3f-8d3f-1093d8d385c1", + "prompt_sha256": "95a3b68ff7aaee81a894f49e8cfbf49f9874b2578840b33e11a3d0cd65f72814", + "verdict_sha256": "f9e281d4620ae229345fd73a1a1d3cb88c1f5ab516fe82141043676f97d74bfd" + }, + { + "case_id": "us__scenario_112__snap", + "login": "pb-judge", + "session_id": "17ee4f4c-5f81-4213-80fc-6ef9c68993ad", + "prompt_sha256": "379654f6324baa1470449fa2a120c0790bed2a3d81a702fcc7ae5977673a12fd", + "verdict_sha256": "ac170d3f4cbc670f941a5c58f3cda35f1e513a9e2cd4ac13666f93547a79bb25" + }, + { + "case_id": "us__scenario_114__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "0419a008-6df3-4aa6-8e0c-36cfcdcd3e4f", + "prompt_sha256": "d8becc4ab482ebf61fd3b9f3225734b03480be064e2f6ac852f867713bc6363b", + "verdict_sha256": "1330438b2d1f63b43b908ba931b410f08e1613df5e3ad00c7e8f9202f7d6975f" + }, + { + "case_id": "us__scenario_114__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "dd089f0f-8ae4-4e2f-a8cb-7137160b910d", + "prompt_sha256": "704f4a71f89ae1cc31c22e019aabcc6733ac38edd6ffa8d92d8f3527fa8b86c9", + "verdict_sha256": "1324cc2803927a67fa537f5370bcf3873c2261eb9318842bddee4f47469cc0ec" + }, + { + "case_id": "us__scenario_115__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "e4b3d2a2-ed72-4434-8d0c-41a8b7590bf7", + "prompt_sha256": "10a710484b385b25501f9c36b0efa3a056388a2f2a2f4a4fe871121f10b2f7e3", + "verdict_sha256": "aeb8191318a24758a799134d73edfffc608d24b7b974a76f3af488e96cc01a24" + }, + { + "case_id": "us__scenario_116__spouse_medicaid_eligible", + "login": "pb-judge", + "session_id": "1e511dac-eaae-4036-bfde-f54011f8de90", + "prompt_sha256": "c61163baf718458e30f529ab3f9d84f8c6fcd32dd25dd0debc327713d83b9da3", + "verdict_sha256": "c71899c630c2f0e9f6cea6d20f3261faf36b427cf5d70c0d32395f1b4e481f19" + }, + { + "case_id": "us__scenario_117__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "e5584564-e4cb-41ef-a1d1-c43ff8369c42", + "prompt_sha256": "09c0552d4e79e0ebf6414adfdbbdff8386bc4ccde56909916a15cf797970ae4f", + "verdict_sha256": "3162677f98080f94f6cb0728beb1dec62f5d1db1c2e2802abff91cf51414a2e2" + }, + { + "case_id": "us__scenario_117__payroll_tax", + "login": "pb-judge", + "session_id": "0765d015-1443-48cf-8ba9-f80910830bec", + "prompt_sha256": "c335e18898b96ffe23ac22315159e03f576dde59837f0d53c1823e526b1c7c16", + "verdict_sha256": "0e50359baa35a7eeca90dda0d4e5001dc4174f82e94c4a7e9b95419c680860eb" + }, + { + "case_id": "us__scenario_117__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "08d22820-8b9b-4b47-b702-6d71c9eb19cd", + "prompt_sha256": "db635693d7bc860de41c06cc4ca468a09d16932913c718264891e5b2b8b9042f", + "verdict_sha256": "798a891a9d7930043fa8daa6edacd7a9b0a55d6070a6d72dfd4e75386f3df08c" + }, + { + "case_id": "us__scenario_118__head_medicaid_eligible", + "login": "pb-judge", + "session_id": "b4b5cda8-1d2b-4d41-8fbe-1525d2163283", + "prompt_sha256": "e4ab3e94a6988cdc765f69cc7521cda5d6deeb91acd4732a887561f9863e2e87", + "verdict_sha256": "4b659d697f6a85093b208c45914fd9e67cc0c04e9c9352bdab973ff2d31e5a8c" + }, + { + "case_id": "us__scenario_118__snap", + "login": "pb-judge", + "session_id": "8c6bd16c-76ba-4f90-acb3-4b1d405f8235", + "prompt_sha256": "f90f975a772bca6957782b86a5f23dc2496c75cc37ca230676f26306de805014", + "verdict_sha256": "9e3071993def3b401623d836b3c5b9f372ab908d7dc02a1c5a5d32db88d5d858" + }, + { + "case_id": "us__scenario_118__ssi", + "login": "pb-judge", + "session_id": "93b1a408-7ad0-4c3b-8270-bd216594a2aa", + "prompt_sha256": "294d6c5c9dccc91c2f8137be0f4a8069dedbaee62f2ebb4966070df98d3305d3", + "verdict_sha256": "229093fb0cf25b2ecb4477f957041bd7aa80cdc7140bcb1f2086ece5a9ef0a30" + }, + { + "case_id": "us__scenario_118__state_refundable_credits", + "login": "pb-judge", + "session_id": "9fbe0429-938d-478a-b8ae-850d5b04355f", + "prompt_sha256": "73de087579ec693e6b08083dc403790ee946b2b2e25c15a1914b88ecca39c9f9", + "verdict_sha256": "3f4e0f7e3d6625e5e4632e0e16d77393e6e264c58b8aa5511a853fc5ec2055f4" + }, + { + "case_id": "us__scenario_119__federal_refundable_credits", + "login": "pb-judge", + "session_id": "f002de7a-96d9-4162-9ff0-2ee0d4117bdd", + "prompt_sha256": "c86b4c5002d60ef35308098bc91028e3720d2fe256cd937d01a8321d4b78e117", + "verdict_sha256": "cfa26c1a66e81f4b3bcb62267481ecd63558efd4d9d79e3937e53b9096828cd5" + }, + { + "case_id": "us__scenario_119__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "411411ca-12f8-4719-8b59-00c8589a2882", + "prompt_sha256": "08c3b45a7808144983f148fe5477465a75d979819c6edee3037f3313e9e73c7b", + "verdict_sha256": "ec826f2d7459dde71206fc346cc4f3cac0c54c68e2bb00297312d56f39b2a444" + }, + { + "case_id": "us__scenario_120__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "a987d25f-ba49-4f7a-b30e-4ae865d3c1cb", + "prompt_sha256": "6d5a80ab6f4dc9596c6bf051a1e79e6e54a51e510f7bf7b9c1127e82d16a7255", + "verdict_sha256": "85b90ac023b424b1ded692bdfcf1264c3057abec7b76a9a9f5e969be2c1d7f1a" + }, + { + "case_id": "us__scenario_120__payroll_tax", + "login": "pb-judge", + "session_id": "f70d4fff-d824-4e72-a820-e3f355a1f30c", + "prompt_sha256": "2450d5a1ff69962c08526521840a45f0c7f84dae8c9387a57fb6fbe9021265d7", + "verdict_sha256": "f593478dcaf6e4ab985215ea0a73b1b1497befaa5550aff74c5e3f079eb6c91d" + }, + { + "case_id": "us__scenario_120__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "7f95428b-fe79-421b-9876-a24198f2b0f1", + "prompt_sha256": "f4210559b88077959775e708c92318c150ffa2a3122e3b52c0b62261db8a031b", + "verdict_sha256": "e423fe128a5dd75e9e5311634511ace9d93026dfae1b11c6279deb11017b2d1c" + }, + { + "case_id": "us__scenario_122__federal_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "433c3ace-8727-4f11-822f-469e7d18f1a3", + "prompt_sha256": "a8665f2b50ca125ae9645b91e160449da5abe95dd954c6aa347a932bd2280eb5", + "verdict_sha256": "1bcad0e4f0b82d90018af01ef69e8f82a88b28e4c8cc57ecba9561a54461e5fd" + }, + { + "case_id": "us__scenario_122__state_income_tax_before_refundable_credits", + "login": "pb-judge", + "session_id": "64f86d6a-1400-4666-9362-381008252688", + "prompt_sha256": "75f588c4d9eb58d82013b0605131ef9aba877b373fb4f090efb382180ba8f9c5", + "verdict_sha256": "0fc5ffe168fd21ce539b89d2bd0df56c99122efa86d0b0de21666624eb2b97bd" + }, + { + "case_id": "us__scenario_123__federal_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "24a6b60a-2029-40a0-958d-8d3461098cb2", + "prompt_sha256": "82221be4970b2d7513bf2939d0aa78f1e6580349c0213fb56e05970a2515fe74", + "verdict_sha256": "4f6f31d6328752b8321dd1bf304c4fc915568398d887aef734e362c41ac99e02" + }, + { + "case_id": "us__scenario_123__local_income_tax", + "login": "pb-judge", + "session_id": "30af49ba-acb1-4575-9efc-842ac7de963b", + "prompt_sha256": "20e77243a50a63813619ff778a75cb454aa55599ed980836cd48febee90e15d0", + "verdict_sha256": "42e5e539f01cd09f5ec915d9614e97528dae7209472e7d05ad0a89e15e0efe04" + }, + { + "case_id": "us__scenario_123__payroll_tax", + "login": "pb-judge", + "session_id": "990e4578-76b9-4e2b-869e-51e35be8578b", + "prompt_sha256": "d21e14fb430142fadc31cfd53c286f4667e1b13cea2947101e69a8fe5e81fbec", + "verdict_sha256": "ef1083a6758848ebdec794e2164697bd0388d395fdac65a8dc01cf22f1579f48" + }, + { + "case_id": "us__scenario_123__state_income_tax_before_refundable_credits", + "login": "claude-18", + "session_id": "6d53f72c-fc9b-4826-a708-5a5641f638a1", + "prompt_sha256": "21d9ebdded056874eb0dfd5b9a639f75fd70fcaac85ec566f255b9a8607b781e", + "verdict_sha256": "ad43142d632889e7c01321c24189a6c1c30fac9148f5d24c28123cb66fa7b12e" + } + ] +} diff --git a/docs/musewonder/seed_recovery.md b/docs/musewonder/seed_recovery.md new file mode 100644 index 00000000..e1b1e7f3 --- /dev/null +++ b/docs/musewonder/seed_recovery.md @@ -0,0 +1,22 @@ +# The audit seed: release 20261010's judged audit, rebuilt + +Release dashboard-data-20261010 (PR #208) judged 232 cases with Opus 5.5 on judge template v1 and carried 445 cases from release 20261006, 677 judged cases in all. Its stage, which held those verdicts, was in the release worktree, and that worktree was removed on 2026-10-10 after the release shipped. The repository kept only hashes: `docs/haiku55/judge_provenance.json` gives each re-judged case's prompt and verdict sha256. + +Each judge was a Claude Code session run by `scripts/run_audit_claude.sh`. Its session transcript was still in the judge login's configuration directory (`~/.policybench-judge/pb-judge-config` or `claude-18-config`, under `projects/`). The runner writes: + +- `prompt.md`: the bytes the judge read, which are the transcript's one user text message; +- `verdict.json`: `json.dumps(structured_output, indent=2, sort_keys=True)`, where the transcript holds the one accepted StructuredOutput call with the same input. + +`scripts/recover_20261010_audit.py` therefore rebuilds both files from the transcripts. + +How the rebuild ran (2026-10-12): + +1. Release 20261006's local seed tree (674 judged cases) must match `docs/haiku55/seed_digest.csv`. It did. +2. It scanned the 288 judge transcripts. For each, it took the one user text message and the one accepted StructuredOutput input. A transcript stands for a case only when both hash to that case's entry in `docs/haiku55/judge_provenance.json`. +3. All 232 cases were found: 193 on the pb-judge login's transcripts and 39 on claude-18's, the provenance record's two groups. No two transcripts gave one case different bytes. +4. It wrote the 232 cases over release 20261006's tree. Each got its prompt, its verdict, its transcript and a sidecar. The sidecar is rebuilt from the provenance entry and the transcript, and is marked `recovered` because the original sidecar was lost. +5. The rebuilt tree's digest must equal release 20261006's digest overlaid with the provenance entries. It did, and that digest is `docs/musewonder/seed_digest.csv` (sha256 `bc7c057e…`). + +`docs/musewonder/seed_recovery.json` lists each recovered case's login and session id. The tests derive the committed digest from the two Haiku 5.5 records alone (`tests/test_finish_musewonder.py`). + +No verdict text is new. Every carried verdict is the one release 20261010 published, and its committed annotations already reflect it. diff --git a/scripts/date_musewonder_judge_verdicts.py b/scripts/date_musewonder_judge_verdicts.py new file mode 100644 index 00000000..64a6d916 --- /dev/null +++ b/scripts/date_musewonder_judge_verdicts.py @@ -0,0 +1,290 @@ +"""Record the verdicts release dashboard-data-20261010's adjudications name. + +The Claude Haiku 5.5 release restated the judge fields of every decision on a +case Claude Haiku 5.5 re-opened (scripts/restate_gpt61sol_adjudications.py), +dropped the decisions on records its engine upgrade regenerated, and added +two waves of decisions: the 2026-10-06 rulings' (d1022, d994) and the engine +upgrade's, on the outputs it newly excludes. Run on the exported stage after +the freeze, this script + +- writes docs/haiku55/judge_verdicts_20261010.json: for each committed + decision on a re-opened case, the case's verdict in the stage, read from + verdict.json and its sha256-bound verdict.meta.json, as + scripts/date_gpt61sol_judge_verdicts.py did for release 20260930; and +- brings reference_audit/2026-09-28/verification/judge_verdicts.json up to + this release, as release 20261006's driver did + (scripts/release_20261006.py build_evidence): the 2026-10-05 wave's release + is now committed (the driver's BASE_COMMIT), so its decisions gain the + verdicts that release published; this release's waves join, uncommitted, + with the UTC day their decisions were written; and each decision of theirs, + new or restated in place from an earlier wave, gains the stage verdict it + reviewed in place of the earlier evidence. + +tests/test_adjudications.py reads both files on machines without the stage. +A second run on the same stage writes the same bytes. + + python scripts/date_haiku55_judge_verdicts.py --stage-dir STAGE +""" + +from __future__ import annotations + +import argparse +import copy +import datetime +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import finish_haiku55 as driver # noqa: E402 +from date_adds0928_judge_verdicts import _evidence # noqa: E402 + +# Named for the release tag's date (dashboard-data-YYYYMMDD). +OUT = ROOT / f"docs/haiku55/judge_verdicts_{driver.RELEASE_TAG[-8:]}.json" +EVIDENCE = ROOT / "reference_audit/2026-09-28/verification/judge_verdicts.json" +# The wave release 20261006 decided, which this release's base commit froze. +FILL_WAVE = "2026-10-05" +# Release 20261006's waves, before this release's. +BASE_WAVES = ["2026-09-05", "2026-09-22", "2026-09-29", FILL_WAVE] + + +def case_id(entry: dict) -> str: + return f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + + +def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(ROOT), *args], check=True, capture_output=True, text=True + ).stdout + + +def bound_verdict(folder: Path) -> tuple[dict, dict]: + raw = (folder / "verdict.json").read_bytes() + meta = json.loads((folder / "verdict.meta.json").read_text()) + driver.require( + meta["verdict_sha256"] == hashlib.sha256(raw).hexdigest(), + f"{folder.name}: the sidecar does not bind this verdict", + ) + return json.loads(raw), meta + + +def judge_disagreements(entry: dict, verdict: dict, meta: dict) -> list[str]: + """Where a decision's judge fields depart from the verdict it names: the + judge, the classes, the UTC day, and the reference flag. The flag may be + set from an earlier run the verdict does not repeat, but only with its + judge_reference_suspect_source saying so.""" + problems = [] + expected = { + "judge_model": meta["judge_model_requested"], + "judge_failure_source": verdict["case_failure_source"], + "judge_failure_subtype": verdict["case_failure_subtype"], + } + for field, value in expected.items(): + if entry.get(field) != value: + problems.append(f"{field} {entry.get(field)!r} != {value!r}") + day = entry.get("judge_rejudged_on") or entry.get("judged_on_utc") + if day != meta["judged_at_utc"][:10]: + problems.append( + f"judged on {day!r}, the verdict on {meta['judged_at_utc'][:10]}" + ) + flagged, now = ( + bool(entry.get("judge_reference_suspect")), + bool(verdict["reference_suspect"]), + ) + if flagged != now and not ( + flagged and not now and entry.get("judge_reference_suspect_source") + ): + problems.append(f"reference flag {flagged} != the verdict's {now}") + return problems + + +def committed_record() -> dict: + return json.loads((driver.ANNOTATIONS / driver.ADJUDICATIONS).read_text()) + + +def restatements(stage: Path, record: dict) -> dict: + """The current verdict of each committed decision on a re-opened case.""" + reopened = { + entry["case_id"] + for entry in json.loads(driver.JUDGE_PROVENANCE.read_text())["verdicts"] + } + cases = {} + for entry in record["adjudications"]: + case = case_id(entry) + if case not in reopened: + continue + verdict, meta = bound_verdict(stage / "audit" / "cases" / case) + wrong = judge_disagreements(entry, verdict, meta) + driver.require( + not wrong, f"{case}: the decision does not name its verdict: {wrong}" + ) + cases[case] = { + "judge_model": meta["judge_model_requested"], + "case_failure_source": verdict["case_failure_source"], + "case_failure_subtype": verdict["case_failure_subtype"], + "reference_suspect": bool(verdict["reference_suspect"]), + "judged_at_utc": meta["judged_at_utc"], + "verdict_sha256": meta["verdict_sha256"], + } + return { + "note": ( + "The verdicts release dashboard-data-20261010's adjudication entries " + "name on the cases Claude Haiku 5.5 re-opened: for each committed " + "decision on such a case, the case's verdict in the exported stage, " + "read by scripts/date_haiku55_judge_verdicts.py from verdict.json and " + "its sha256-bound verdict.meta.json. The verdict each restatement " + "replaced is release 20261006's entry (git show " + f"{driver.BASE_COMMIT[:12]}:annotations/{driver.RUN_NAME}/" + f"{driver.ADJUDICATIONS})." + ), + "release": driver.RELEASE_TAG, + "base_commit": driver.BASE_COMMIT, + "cases": dict(sorted(cases.items())), + } + + +def fill_release() -> dict: + """The 2026-10-05 wave's release as committed: the base commit, its pull + request (the squash subject's "(#N)") and its UTC commit day.""" + subject = git("show", "-s", "--format=%s", driver.BASE_COMMIT).strip() + number = re.search(r"\(#(\d+)\)$", subject) + driver.require(number is not None, f"{subject!r} names no pull request") + when = datetime.datetime.fromisoformat( + git("show", "-s", "--format=%cI", driver.BASE_COMMIT).strip() + ) + return { + "commit": driver.BASE_COMMIT, + "release": "dashboard-data-20261006", + "pull_request": int(number.group(1)), + "committed_on": when.astimezone(datetime.timezone.utc).date().isoformat(), + } + + +def upgrade_day() -> str | None: + """The day of the committed sidecar's engine upgrade, if this release has one + (the last revision, after release 20261006's 2026-09-29 move).""" + meta = json.loads((driver.SNAPSHOT / "reference_outputs.csv.meta.json").read_text()) + last = meta["revisions"][-1] + if last.get("kind") == "engine_upgrade" and last["date"] > BASE_WAVES[-1]: + return last["date"] + return None + + +def new_waves(record: dict, upgrade: str | None = None) -> dict[str, str]: + """This release's waves and the UTC day each one's decisions were written + (the spec's adjudications_written_on for the 2026-10-06 rulings, its + upgrade_adjudications_written_on for the engine upgrade's wave). The waves + must be the 2026-10-06 rulings' and, with an engine upgrade, its day.""" + spec = driver.load_spec() + waves = sorted( + {e["adjudicated_on"] for e in record["adjudications"]} - set(BASE_WAVES) + ) + allowed = {"2026-10-06"} | ({upgrade} if upgrade else set()) + driver.require( + "2026-10-06" in waves and set(waves) <= allowed, + f"this release's waves are {waves}, not the 2026-10-06 rulings' and the " + f"engine upgrade's ({sorted(allowed)})", + ) + written = {} + for wave in waves: + if wave == "2026-10-06": + written[wave] = spec["adjudications_written_on"] + else: + written[wave] = spec["upgrade_adjudications_written_on"] + driver.require(wave <= written[wave], f"the {wave} wave was written before it") + return written + + +def base_wave_releases(waves: dict) -> dict: + """Release 20261006's waves alone. The waves after them are this release's, + which every pass writes afresh from the record, so an earlier pass's (a + rehearsal's, or one whose record has since changed) cannot outlive it.""" + driver.require( + list(waves)[: len(BASE_WAVES)] == BASE_WAVES, + "the evidence's waves are not release 20261006's", + ) + return {wave: waves[wave] for wave in BASE_WAVES} + + +def updated_evidence(evidence: dict, stage: Path, record: dict) -> dict: + evidence = copy.deepcopy(evidence) + waves = base_wave_releases(evidence["wave_releases"]) + fill = fill_release() + if waves[FILL_WAVE].get("commit") is None: + driver.require( + waves[FILL_WAVE]["release"] == fill["release"], + f"the {FILL_WAVE} wave is not release 20261006's", + ) + waves[FILL_WAVE] = fill + driver.require( + waves[FILL_WAVE] == fill, f"the {FILL_WAVE} wave was filled otherwise" + ) + published = json.loads( + git( + "show", + f"{driver.BASE_COMMIT}:annotations/{driver.RUN_NAME}/{driver.ADJUDICATIONS}", + ) + )["adjudications"] + filled = 0 + for entry in published: + if entry["adjudicated_on"] != FILL_WAVE: + continue + evidence["cases"][case_id(entry)]["published"] = { + "release": fill["release"], + "commit": fill["commit"], + "judge_model": entry["judge_model"], + "case_failure_source": entry["judge_failure_source"], + "case_failure_subtype": entry["judge_failure_subtype"], + "judged_on_utc": entry.get("judged_on_utc"), + } + filled += 1 + driver.require(filled > 0, f"release 20261006 published no {FILL_WAVE} decisions") + for wave, written in new_waves(record, upgrade_day()).items(): + waves[wave] = { + "commit": None, + "release": driver.RELEASE_TAG, + "pull_request": None, + "adjudications_written_on": written, + } + for entry in record["adjudications"]: + if entry["adjudicated_on"] == wave: + case = case_id(entry) + folder = stage / "audit" / "cases" / case + # A decision this wave restated in place (the spec's + # restated_adjudications) was an earlier wave's: its case's + # evidence now describes this wave's decision, which no release + # has published yet. The earlier release's evidence stays in + # git at that release's commit, where its tests read it. + verdict, meta = bound_verdict(folder) + wrong = judge_disagreements(entry, verdict, meta) + driver.require( + not wrong, + f"{case}: the decision does not name its verdict: {wrong}", + ) + found = evidence["cases"].setdefault(case, {}) + found["current"] = _evidence("stage", (verdict, meta)) + found.pop("published", None) + evidence["wave_releases"] = dict(sorted(waves.items())) + return evidence + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + args = parser.parse_args(argv) + record = committed_record() + OUT.write_text(json.dumps(restatements(args.stage_dir, record), indent=1) + "\n") + evidence = updated_evidence( + json.loads(EVIDENCE.read_text()), args.stage_dir, record + ) + EVIDENCE.write_text(json.dumps(evidence, indent=2) + "\n") + print(f"Wrote {OUT.relative_to(ROOT)} and {EVIDENCE.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py new file mode 100644 index 00000000..e6fa148a --- /dev/null +++ b/scripts/finish_musewonder.py @@ -0,0 +1,5010 @@ +"""Stage the Meta Muse Wonder addition; never publish or change tracked data. + +Adapted from finish_haiku55.py. It folds one supervised run onto the 47-model +board of release dashboard-data-20261010 and moves the references from +policyengine-us 2.38.6 to the newest release (Max, 2026-09-28: "we should be +using the latest pe for this always!"), regenerating the excluded outputs whose +engine defects that release fixes (--step install-references; see "An engine +upgrade of the references" below). The release rules no new exclusion, so +docs/musewonder/spec.json lists no proposals and --step install-exclusions +installs release 20261010's record unchanged. All outputs, including audit +verdicts and adjudications, stay in --stage-dir. See docs/musewonder/design.md +for the run and release procedure. + +The seed is release 20261010's judged audit, rebuilt from the judges' session +transcripts after the release's stage was deleted with its worktree +(docs/musewonder/seed_recovery.md): every judged case keeps the v1 prompt and +verdict its judge wrote, and new and re-opened cases render on judge template +v2 (JUDGE_TEMPLATE_VERSION). +""" + +from __future__ import annotations + +import argparse +import contextlib +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +os.environ["OPENBLAS_NUM_THREADS"] = "1" +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +SNAPSHOT = ROOT / "paper/snapshot/20260501/runs" / RUN_NAME +ANNOTATIONS = ROOT / "annotations" / RUN_NAME +BASE_TAG = "dashboard-data-20261010" +# The one place the new release's tag is named. The lead may change it at +# freeze time; freeze_musewonder.py and the design note read it from here. +RELEASE_TAG = "dashboard-data-20261013" +BASE_SHA256 = "f834478e65191f8426ef479506f9f6fe32c2e7107f30fdbf4e7ba0e6acba6410" +BASE_MODELS = 47 +BASE_OUTPUTS = 1984 +BASE_EXCLUSIONS = 58 +BASE_SCORED = BASE_OUTPUTS - BASE_EXCLUSIONS +# Ruled exclusions, installed by --step install-exclusions and adjudicated by +# --step adjudicate-exclusions, as docs/musewonder/spec.json lists them. This +# release rules none: d1252 (VA 039, OH 025) and d1263 (the Tax Table) were +# still open when it was staged, and it changes nothing scored on their points. +SPEC_PATH = Path("docs/musewonder/spec.json") +NEW_EXCLUSIONS = 0 +RELEASE_EXCLUSIONS = BASE_EXCLUSIONS + NEW_EXCLUSIONS +RELEASE_SCORED = BASE_OUTPUTS - RELEASE_EXCLUSIONS +MODELS = {"musewonder": "muse-wonder"} +BOARD_MODELS = BASE_MODELS + len(MODELS) +REFERENCE_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", + "us_adjudications.json", +) +JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on: v2, which drops v1's claim that the reference +# pipeline survived an adversarial review with its real bugs fixed (#210). A +# seed case keeps the version its sidecar records (v1 for every case release +# 20261010 judged), so the stage's audit mixes the two. +JUDGE_TEMPLATE_VERSION = 2 +ADJUDICATIONS = "us_adjudications.json" +# Stage files export binds: the cases Muse Wonder re-opened, and the +# wording-only amendments a developer lists for them. +PROMPT_CHANGES = "prompt-changes.json" +AMENDMENTS = "wording-amendments.json" +# The published record of how each new verdict (a case Muse Wonder re-opened) +# was judged. Export requires it to describe the staged verdicts and binds its +# bytes in the receipt. +JUDGE_PROVENANCE_PATH = "docs/musewonder/judge_provenance.json" +JUDGE_PROVENANCE = ROOT / JUDGE_PROVENANCE_PATH +# The record's keys, and each entry's: every one is checked, against the +# case's verdict, prompt or sidecar, the entries' tally, or the isolated flag. +JUDGE_PROVENANCE_KEYS = frozenset({"note", "counts", "verdicts"}) +JUDGE_PROVENANCE_ENTRY_KEYS = frozenset( + { + "case_id", + "group", + "isolated", + "judge_account_declared", + "judge_effort", + "judge_model_reported", + "judged_at_utc", + "prompt_sha256", + "verdict_sha256", + } +) +# The entry fields copied from the verdict's sidecar, an e-mail address in any +# of them withheld as WITHHELD_ADDRESS: the record is public. +JUDGE_SIDECAR_FIELDS = ( + "judge_effort", + "judge_model_reported", + "judged_at_utc", + "judge_account_declared", +) +WITHHELD_ADDRESS = "" +# The context attachments an isolated judge's transcript may carry: ATTACHMENTS +# in scripts/run_audit_claude.sh, which a test keeps equal to this set. +JUDGE_ATTACHMENTS = frozenset( + { + "environment", + "model", + "date", + "session_context", + "total_tokens_reminder", + "prompt_snapshot", + "structured_output", + "silent_turn_reminder", + } +) +# The event types an isolated judge's transcript may carry, the types the +# stage's 60 isolated transcripts carry: EVENT_TYPES in +# scripts/run_audit_claude.sh, which a test keeps equal to this set. +JUDGE_EVENT_TYPES = frozenset( + { + "queue-operation", + "user", + "attachment", + "atis-latch", + "last-prompt", + "assistant", + "cost-state", + } +) +# Account data an isolated judge's transcript may not carry outside the judged +# prompt's own user message and the judge's own words: an e-mail address in any +# key or string, or a key that names an account, at any depth. The runner's +# extract_verdict uses the same two patterns. +ACCOUNT_KEY = re.compile(r"email|account|credential|organi[sz]ation|gitstatus", re.I) +EMAIL_ADDRESS = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") +# Claude Code's own nudge to a judge that answered in text, the one user text +# message an isolated judge's transcript may carry besides its prompt: +# PROMPT_NUDGE in scripts/run_audit_claude.sh, which a test keeps equal to this. +JUDGE_PROMPT_NUDGE = ( + "[structured-output-enforce] You MUST call the StructuredOutput tool to " + "complete this request. Call this tool now." +) +# The merge of PR #208 on main, whose tree holds release 20261010. +BASE_COMMIT = "5a8164a001efb27fa55f47fe7ea26666a0de31f8" +# Release 20261010's references (policyengine-us 2.38.6). Prepare stages +# them unchanged; --step install-references replaces them with the engine +# upgrade's build. +BASE_REFERENCE_SHA256 = { + "reference_outputs.csv": ( + "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371" + ), + "reference_outputs.csv.meta.json": ( + "d534cfeacf7f8a8b7f6fca477adbb02c5bb52b226778d63cc10fd117ec70c54b" + ), + "reference_exclusions.json": ( + "089e01e1f699aa19f2171e1ac5b6dc2ce775f9898c94420e6a0b06ff5f773626" + ), + "scenarios.csv": ( + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a" + ), + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +# The grounding the 20260929 stage (adds0928-v3) rendered into its prompts, and +# the GPT-6.1 Sol and 20261010 stages after it: +# results/local/unified_audit/grounding.csv in the main clone. Every seed +# prompt re-renders byte-identically from it and the committed snapshot, so a +# different grounding would silently invalidate carried-over verdicts. +GROUNDING_SHA256 = "b1e4a9bc74d762f410524a147efcda7d705c3afcfa3dc27f720fa60c54a7b55c" +# The seed: every judged case of release 20261010's audit, with the sha256 of +# its prompt and of its verdict, committed at docs/musewonder/seed_digest.csv: +# release 20261006's 674 judged cases (docs/haiku55/seed_digest.csv) with the +# 232 that release 20261010 re-judged (docs/haiku55/judge_provenance.json) +# written over them, 677 in all. This pins that file's bytes; prepare refuses +# any other seed and binds this one in stage.json, and a carried-over verdict +# must keep the seed's bytes. +SEED_DIGEST = ROOT / "docs/musewonder/seed_digest.csv" +SEED_DIGEST_SHA256 = "bc7c057e58c4fa8425fa138c57e6b6a2f5cbeba6f4e304ea0438c8c67e2eb883" +# Muse Wonder's supervised run as it finished, pinned outside the stage: the +# sha256 of the run directory's predictions.csv and run_state.json, which +# --step pin-inputs writes here. Export and the freeze read the file as +# committed at HEAD, and the stage's inputs// copies must be those bytes. +INPUT_PINS_PATH = "docs/musewonder/input_pins.json" +INPUT_PINS = ROOT / INPUT_PINS_PATH +PINNED_INPUTS = ("predictions.csv", "run_state.json") +# Incumbent usage the exporter cannot recompute from committed predictions: +# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, +# token or latency fields, so export_full_run reports $0 and omits the rest. +# The released values are carried over; the drift gate then checks every field. +CARRIED_USAGE = { + "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") +} +KEY = ["scenario_id", "variable"] +# The tree whose committed annotations render every seed prompt: release +# 20261010's own (PR #208's merge). Its 445 carried cases and 232 re-judged +# ones all re-render from it on v1 (tests/test_judge_template.py's +# committed-prompt invariant), so this is BASE_COMMIT. +SEED_RELEASE_COMMIT = BASE_COMMIT +REFERENCE_EXPLANATIONS = ( + Path("annotations") / RUN_NAME / "us_case_reference_explanations.csv" +) +# The cases whose reference explanation changed between the seed's rendering +# and the base. Both are release 20261010, so none: reworded_since_seed() +# derives the set from git and must find it empty. +REWORDED_SINCE_SEED: frozenset[str] = frozenset() + + +def require(condition: bool, message: str) -> None: + """Fail before an unsafe or incomplete stage can proceed.""" + if not condition: + raise SystemExit(message) + + +def digest(path: Path) -> str: + """Hash a file without retaining its contents in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def write_json(path: Path, value: object) -> None: + """Write a deterministic, finite JSON artifact.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + + +def verify_reference_pins( + directory: Path, + label: str, + exclusions_sha256: str | None = None, + pins: dict[str, str] | None = None, +) -> None: + """Every reference file in ``directory`` must equal its 20261010 pin; the + exclusion record may instead be the release's (``exclusions_sha256``). + + ``pins`` replaces the 20261010 pins outright: an installed engine upgrade + names its build's hashes (upgraded_reference_pins).""" + for name, pin in (BASE_REFERENCE_SHA256 if pins is None else pins).items(): + if name == "reference_exclusions.json" and exclusions_sha256 is not None: + pin = exclusions_sha256 + path = directory / name + require( + path.is_file() and digest(path) == pin, + f"{label} {name} does not match its pin; " + + ( + "this release revises no reference value" + if pins is None + else "it is not the installed engine upgrade's build" + ), + ) + + +@dataclass(frozen=True) +class NewRun: + slug: str + model: str + run_dir: Path + predictions: Path + state: dict + + +def discover_new_models(runs_root: Path) -> list[NewRun]: + """Require every named run to be completed and unstopped, as before.""" + found = [] + skipped = [] + for slug, expected in MODELS.items(): + directory = runs_root / slug / "run" + state_path = directory / "run_state.json" + if not state_path.is_file(): + skipped.append(f"{slug}: no run_state.json") + continue + state = json.loads(state_path.read_text()) + completed, total = state.get("completed"), state.get("total") + reason = state.get("stopped_reason") + predictions = directory / "predictions.csv" + if ( + type(total) is not int + or type(completed) is not int + or total <= 0 + or completed != total + or reason is not None + ): + skipped.append(f"{slug}: {completed}/{total}, stopped_reason={reason!r}") + elif state.get("model") != expected: + skipped.append(f"{slug}: unexpected model {state.get('model')!r}") + elif not predictions.is_file(): + skipped.append(f"{slug}: no predictions.csv") + else: + found.append(NewRun(slug, expected, directory, predictions, state)) + require(not skipped, "refusing incomplete additions: " + "; ".join(skipped)) + return found + + +def input_pins_record(runs_root: Path) -> dict: + """The record --step pin-inputs writes: the sha256 of each new model's + finished run files, read from the run directory under ``runs_root``.""" + return { + "note": ( + "The sha256 of Muse Wonder's supervised run files as the run " + "finished, written by scripts/finish_musewonder.py --step pin-inputs " + "from the run directory. prepare copies them into the stage's " + "inputs//; export and the freeze refuse a staged copy that " + "is not these bytes, and read this file as committed at HEAD." + ), + "inputs": { + run.slug: { + "run": f"{runs_root.name}/{run.slug}/run", + "sha256": {name: digest(run.run_dir / name) for name in PINNED_INPUTS}, + } + for run in discover_new_models(runs_root) + }, + } + + +def pin_inputs(args) -> None: + """Write INPUT_PINS from the finished run; commit it before export.""" + write_json(INPUT_PINS, input_pins_record(args.runs_root)) + print(f"Pinned {', '.join(PINNED_INPUTS)} of {sorted(MODELS)} in {INPUT_PINS_PATH}") + + +def head_blob(path: str) -> bytes: + """A repository file as committed at HEAD, never the working tree's copy, + which could be edited together with the stage it pins.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"HEAD:{path}"], capture_output=True + ) + require( + result.returncode == 0, + f"{path} is not committed at HEAD; commit it: {result.stderr.decode().strip()}", + ) + return result.stdout + + +def committed_input_pins() -> dict[str, dict[str, str]]: + """Each new model's pinned run-file sha256, as committed at HEAD. + + The working-tree file must be HEAD's too, so pins written and not yet + committed are refused rather than silently ignored. + """ + blob = head_blob(INPUT_PINS_PATH) + require( + INPUT_PINS.is_file() and INPUT_PINS.read_bytes() == blob, + f"{INPUT_PINS_PATH} differs from its HEAD commit; commit it", + ) + record = json.loads(blob) + inputs = record.get("inputs") if isinstance(record, dict) else None + require( + isinstance(inputs, dict) + and set(inputs) == set(MODELS) + and all( + isinstance(item, dict) + and isinstance(item.get("sha256"), dict) + and set(item["sha256"]) == set(PINNED_INPUTS) + for item in inputs.values() + ), + f"{INPUT_PINS_PATH} does not pin {list(PINNED_INPUTS)} for {sorted(MODELS)}", + ) + return {slug: item["sha256"] for slug, item in inputs.items()} + + +def verify_input_pins(stage: Path) -> None: + """The stage's copies of each new model's run files are the pinned bytes.""" + for slug, pins in committed_input_pins().items(): + for name, pin in pins.items(): + path = stage / "inputs" / slug / name + require( + path.is_file() and digest(path) == pin, + f"staged inputs/{slug}/{name} is not the run file " + f"{INPUT_PINS_PATH} pins; prepare a new stage from the pinned run", + ) + + +def verify_prepared_inputs(stage: Path) -> None: + """The prepare-time hashes in stage.json and model-provenance.json hold. + + stage.json records every file prepare copied or folded (each new model's + run files, the bundle's predictions and references); model-provenance.json + records each new model's predictions sha256 and treatment fingerprint. + """ + receipt = json.loads((stage / "stage.json").read_text()) + files = receipt.get("files") if isinstance(receipt, dict) else None + require(isinstance(files, dict), "stage.json records no prepared files") + us = Path("publish") / RUN_NAME / "us" + expected = {str(us / name) for name in (*REFERENCE_FILES, "predictions.csv")} + expected |= { + str(Path("inputs") / slug / name) for slug in MODELS for name in PINNED_INPUTS + } + require( + expected <= set(files), + f"stage.json does not record {sorted(expected - set(files))}", + ) + for name, pin in files.items(): + path = (stage / name).resolve() + require( + path.is_relative_to(stage.resolve()) + and path.is_file() + and digest(path) == pin, + f"staged input changed since prepare: {name}", + ) + provenance = json.loads((stage / "model-provenance.json").read_text()) + require( + isinstance(provenance, dict) and set(provenance) == set(MODELS.values()), + f"model-provenance.json does not describe exactly {sorted(MODELS.values())}", + ) + for slug, model in MODELS.items(): + inputs = stage / "inputs" / slug + state = json.loads((inputs / "run_state.json").read_text()) + entry = provenance[model] + fingerprint = state.get("treatment_fingerprint") + require( + isinstance(entry, dict) + and entry.get("predictions_sha256") == digest(inputs / "predictions.csv") + and entry.get("treatment_fingerprint") == fingerprint, + f"model-provenance.json disagrees with the staged run of {model}", + ) + + +def new_model_row_differences(stage: Path, limit: int = 5) -> list[str]: + """Where a new model's rows in the bundle's predictions are not its run's. + + Every column of the staged run file (inputs//predictions.csv, which + verify_input_pins pins) is compared cell by cell, as text, keyed by + scenario and variable: the answers and also the cost, token and latency + columns the payload's costUsd, totalTokens and latencySeconds come from. + No column is exempt. prepare_inputs copies the run file, and fold_board + reads it with read_csv and writes its concat with the base by to_csv. + That round trip could rewrite a cell (an integer column the base leaves + blank would come back as a float), and such a cell is refused, never + skipped; on the real stage it changes no cell of the run's 26 columns + (its token and cost columns are already floats). A bundle column the run + lacks (a base column the concat adds) must be empty on the new model's + rows. + """ + import pandas as pd + + text = {"dtype": str, "keep_default_na": False} + us = stage / "publish" / RUN_NAME / "us" / "predictions.csv" + header = list(pd.read_csv(us, nrows=0, **text).columns) + if "model" not in header: + return [f"{us.name} has no model column"] + parts: dict[str, list] = {} + for chunk in pd.read_csv(us, chunksize=100_000, **text): + for model in MODELS.values(): + rows = chunk[chunk["model"] == model] + if len(rows): + parts.setdefault(model, []).append(rows) + found = [] + for slug, model in MODELS.items(): + source = pd.read_csv(stage / "inputs" / slug / "predictions.csv", **text) + folded = ( + pd.concat(parts[model]) if model in parts else pd.DataFrame(columns=header) + ) + missing = [column for column in source.columns if column not in header] + if missing or not set(KEY) <= set(source.columns): + found.append(f"{model}: run columns {missing or KEY} not in the bundle") + continue + run = source.sort_values(KEY, kind="stable").reset_index(drop=True) + board = folded.sort_values(KEY, kind="stable").reset_index(drop=True) + if ( + run.duplicated(KEY).any() + or len(run) != len(board) + or not run[KEY].equals(board[KEY]) + ): + found.append( + f"{model}: the bundle holds {len(board)} rows, not the run's " + f"{len(run)} scenario and variable keys" + ) + continue + for column in header: + if column in run.columns: + differ = run[column] != board[column] + what = "differs from the run" + else: + differ = board[column] != "" + what = "is filled, and the run has no such column" + if differ.any(): + first = board.loc[differ.idxmax(), KEY] + found.append( + f"{model} {column} {what} on {int(differ.sum())} rows " + f"(first {'__'.join(first)})" + ) + return found[:limit] + + +def verify_new_model_inputs(stage: Path) -> None: + """Each new model's staged inputs are its pinned run, as prepare folded it. + + Its staged run files are the bytes INPUT_PINS (committed at HEAD) records, + the prepare-time hashes in stage.json and model-provenance.json still + hold, and its rows in the bundle's predictions are its run file's, every + column, so its published cost, tokens and latency are the run's. + """ + verify_input_pins(stage) + verify_prepared_inputs(stage) + differ = new_model_row_differences(stage) + require( + not differ, + f"{sorted(MODELS.values())} rows in the bundle's predictions are not the " + f"pinned run's: {differ}; prepare a new stage", + ) + + +def validate_stage_path(stage: Path, sources: list[Path]) -> None: + """Allow only scratch outputs, disjoint from every read-only source.""" + stage = stage.resolve() + scratch = (ROOT / "results/local").resolve() + require(scratch in stage.parents, f"stage-dir must be below {scratch}") + for source in sources: + source = source.resolve() + require( + source != stage + and source not in stage.parents + and stage not in source.parents, + f"stage-dir overlaps input: {source}", + ) + if stage.exists(): + require( + not any(p.is_symlink() for p in stage.rglob("*")), "symlink in stage-dir" + ) + + +def validate_keys(frame, reference, model: str) -> None: + """Row count alone cannot detect substituted or missing output keys.""" + require(set(frame.model) == {model}, f"unexpected model rows for {model}") + require(not frame.duplicated(KEY).any(), f"duplicate prediction keys: {model}") + expected = set(reference[KEY].itertuples(index=False, name=None)) + actual = set(frame[KEY].itertuples(index=False, name=None)) + require(actual == expected, f"prediction keys differ from reference: {model}") + + +def live_pointer() -> dict: + """The committed dashboard pointer (release 20261010 until the freeze).""" + return json.loads((ROOT / "app/src/data.artifact.json").read_text()) + + +def resolve_base(args): + """Pin the live payload and its committed predictions/reference bundle.""" + import pandas as pd + + # Pandas 3 infers Arrow strings and expands repeated full raw responses + # into large buffers. Object strings retain the CSV parser's deduplication, + # as in the Pandas 2 environment used for the previous release. + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + verify_reference_pins(SNAPSHOT, "committed reference") + pointer = live_pointer() + require( + pointer["tag"] == BASE_TAG and pointer["sha256"] == BASE_SHA256, + "base pointer changed; review the base before staging", + ) + opener = gzip.open if args.base_payload.suffix == ".gz" else open + with opener(args.base_payload, "rb") as stream: + raw = stream.read() + live = json.loads(raw) + if "countries" not in live: + # The compact snapshot freezes the country payload, not its wrapper. + live = {"countries": {"us": live}} + raw = json.dumps(live).encode() + require( + hashlib.sha256(raw).hexdigest() == BASE_SHA256, "base payload SHA256 mismatch" + ) + require( + len(live["countries"]["us"]["modelStats"]) == BASE_MODELS, + f"base must have {BASE_MODELS} models", + ) + + # An explicit alternative is allowed only if it has identical CSV bytes. + def csv_digest(path): + opener = gzip.open if path.suffix == ".gz" else open + h = hashlib.sha256() + with opener(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + verify_base_predictions(SNAPSHOT) + if args.base_predictions.resolve() != (SNAPSHOT / "predictions.csv.gz").resolve(): + require( + csv_digest(args.base_predictions) + == csv_digest(SNAPSHOT / "predictions.csv.gz"), + "base predictions differ from committed 20261010 snapshot", + ) + print("Reading pinned base predictions", flush=True) + # Rehearsals do not need repeated raw provider transcripts for incumbents. + # Their full original CSV remains hash-pinned and untouched in the snapshot. + usecols = (lambda column: column != "raw_response") if args.partial else None + base = pd.read_csv(args.base_predictions, low_memory=True, usecols=usecols) + reference = pd.read_csv(SNAPSHOT / "reference_outputs.csv") + require( + len(reference) == BASE_OUTPUTS and reference.scenario_id.nunique() == 100, + "unexpected reference universe", + ) + require( + base.model.nunique() == BASE_MODELS, + f"base predictions must have {BASE_MODELS} models", + ) + for model, frame in base.groupby("model"): + validate_keys(frame, reference, model) + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + exclusions = load_reference_exclusions(SNAPSHOT) + require( + len(exclusions) == BASE_EXCLUSIONS, f"expected {BASE_EXCLUSIONS} exclusions" + ) + require( + len(split_reference(reference, exclusions)[0]) == BASE_SCORED, + f"expected {BASE_SCORED} scored outputs", + ) + return base, reference, live + + +def base_commit_blob(path: Path) -> bytes: + """A file as committed at BASE_COMMIT, whose tree holds release 20261010. + + The checkout must hold that commit: a shallow clone (CI's default) does + not, so the CI test job checks out full history. + """ + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{BASE_COMMIT}:{path.as_posix()}"], + capture_output=True, + ) + require( + result.returncode == 0, + f"cannot read {path} at base commit {BASE_COMMIT[:12]}; fetch full " + f"history (git fetch --unshallow): {result.stderr.decode().strip()}", + ) + return result.stdout + + +def verify_base_predictions(snapshot: Path) -> None: + """The snapshot's predictions are release 20261010's. + + The pin is that release's own: the snapshot manifest as committed at + BASE_COMMIT, not the working tree's, which could be edited together with + the predictions it pins. + """ + manifest = json.loads( + base_commit_blob(Path("paper/snapshot/20260501/manifest.json")) + ) + pin = manifest["source_run_artifacts"][RUN_NAME]["files"]["predictions.csv.gz"] + require( + digest(snapshot / "predictions.csv.gz") == pin, + "committed base predictions fail release 20261010's manifest hash", + ) + + +def base_payload_from_commit() -> dict: + """Release 20261010's payload, read from BASE_COMMIT and checked.""" + blob = base_commit_blob((SNAPSHOT / "data.json.gz").relative_to(ROOT)) + live = {"countries": {"us": json.loads(gzip.decompress(blob))}} + require( + hashlib.sha256(json.dumps(live).encode()).hexdigest() == BASE_SHA256, + "base payload SHA256 mismatch", + ) + require( + len(live["countries"]["us"]["modelStats"]) == BASE_MODELS, + f"base must have {BASE_MODELS} models", + ) + return live + + +def base_adjudication_record() -> dict: + """Release 20261010's adjudication record file, read from BASE_COMMIT. + + The working-tree copy is not a baseline: the freeze overwrites it, and a + freeze that stops partway would leave the staged record in its place. + """ + path = Path("annotations") / RUN_NAME / ADJUDICATIONS + return json.loads(base_commit_blob(path)) + + +def base_adjudications() -> list[dict]: + """Release 20261010's adjudication entries, read from BASE_COMMIT.""" + from policybench.adjudications import parse_adjudications + + path = Path("annotations") / RUN_NAME / ADJUDICATIONS + return parse_adjudications(base_adjudication_record(), f"{BASE_COMMIT[:12]}:{path}") + + +# --- Ruled exclusions (none in this release) --------------------------------- + +EXCLUSION_REASON_CODES = { + "reference_engine_defect": ("reference_engine_defect", "engine_defect"), + "reference_depends_on_unlisted_input": ("prompt_ambiguity", "unlisted_input"), + "reference_law_published_after_freeze": ("reference_later_law", "later_law"), +} +# Release 20261010's date conventions: the waves it names, and the end of the +# sentence on its last wave, after which a new wave's sentence goes. +DATE_CONVENTIONS_BASE = ( + "adjudicated_on names the audit wave that made the decision (2026-09-05, " + "2026-09-22, 2026-09-29, 2026-10-05, 2026-10-06 or 2026-10-10)." +) +DATE_CONVENTIONS_LAST_WAVE = " or 2026-10-10)" +DATE_CONVENTIONS_ANCHOR = ( + "moved onto an input the prompt does not state, and were written on " + "2026-10-10 UTC." +) + + +def load_spec() -> dict: + return json.loads((ROOT / SPEC_PATH).read_text()) + + +def spec_key(item: dict) -> tuple[str, str]: + return item["scenario_id"], item["variable"] + + +def spec_records(spec: dict) -> list[dict]: + """The ruled records, exactly the outputs each proposal names, in + (scenario_id, variable) order; each proposal file must be the bytes the + spec pins.""" + import copy + + records = [] + for proposal in spec["proposals"]: + path = ROOT / proposal["path"] + require( + digest(path) == proposal["sha256"], + f"{proposal['path']} is not the file the spec pins", + ) + doc = json.loads(path.read_text()) + if proposal["records"] == "root_causes.*.exclusions": + found = [ + record + for cause in doc["root_causes"].values() + for record in cause["exclusions"] + ] + else: + require(proposal["records"] == "exclusions", "unknown record pointer") + found = doc["exclusions"] + named = sorted(tuple(output) for output in proposal["outputs"]) + chosen = [record for record in found if spec_key(record) in set(named)] + require( + sorted(spec_key(record) for record in chosen) == named, + f"{proposal['path']} does not hold exactly the ruled outputs {named}", + ) + records.extend(copy.deepcopy(chosen)) + keys = [spec_key(record) for record in records] + require(len(set(keys)) == len(keys), f"two ruled records share an output: {keys}") + require(len(records) == NEW_EXCLUSIONS, f"expected {NEW_EXCLUSIONS} records") + for record in records: + require( + record["reason_code"] in EXCLUSION_REASON_CODES + and record["decided_on"] == spec["decided_on"] + and record["decided_by"] == "developer" + and record["engine_version"] == BASE_ENGINE, + f"{spec_key(record)} is not a {spec['decided_on']} developer record on " + f"{BASE_ENGINE}", + ) + return sorted(records, key=spec_key) + + +def base_exclusion_record() -> dict: + """Release 20261010's exclusion record, read from BASE_COMMIT and pinned.""" + blob = base_commit_blob((SNAPSHOT / "reference_exclusions.json").relative_to(ROOT)) + require( + hashlib.sha256(blob).hexdigest() + == BASE_REFERENCE_SHA256["reference_exclusions.json"], + "release 20261010's exclusion record does not match its pin", + ) + return json.loads(blob) + + +def build_release_exclusions(base: dict, spec: dict) -> dict: + """Release 20261010's record plus the spec's ruled records (none here). + + The new records go after the earlier waves' and before the audit's + trailing scenario_023 record (tests/test_reference_upgrade.py requires that + tail); the derivation's new sentence goes before the 2026-09-29 marker, as + release 20261010's did. Every earlier record keeps its bytes. + """ + import copy + import tempfile + + from policybench.reference_exclusions import load_reference_exclusions + + doc = copy.deepcopy(base) + new = spec_records(spec) + if not new: + # Nothing ruled: the release's record is release 20261010's, and the + # spec may carry no edit or derivation sentence for records it lacks. + require( + not spec.get("record_edits") and not spec.get("derivation_insert"), + "the spec edits records or the derivation but rules no record", + ) + return doc + tail = doc["exclusions"][-1] + require( + spec_key(tail) == ("scenario_023", "head_medicaid_eligible"), + "release 20261010's record does not end with the audit's scenario_023 record", + ) + existing = {spec_key(record) for record in doc["exclusions"]} + require( + not existing & {spec_key(record) for record in new}, + "a ruled output is already excluded", + ) + by_key = {spec_key(record): record for record in new} + for edit in spec.get("record_edits", []): + record = by_key.get(spec_key(edit)) + require(record is not None, f"no ruled record for edit {spec_key(edit)}") + text = record[edit["field"]] + require( + text == edit["replace"], + f"{spec_key(edit)} {edit['field']} is not the text the edit replaces", + ) + record[edit["field"]] = edit["with"] + doc["exclusions"] = doc["exclusions"][:-1] + new + [tail] + insert = spec["derivation_insert"] + require( + doc["derivation"].count(insert["before"]) == 1, + "the derivation does not hold its 2026-09-29 marker exactly once", + ) + doc["derivation"] = doc["derivation"].replace( + insert["before"], insert["text"] + insert["before"] + ) + with tempfile.TemporaryDirectory() as scratch: + # The loader is the record's one validator: reason codes, required + # fields, numeric values that differ, and no output listed twice. + (Path(scratch) / "reference_exclusions.json").write_text(exclusions_text(doc)) + loaded = load_reference_exclusions(Path(scratch)) + require(len(loaded) == RELEASE_EXCLUSIONS, "wrong exclusion count") + return doc + + +def exclusions_text(doc: dict) -> str: + """The exclusion record as the reference builder spells it.""" + return json.dumps(doc, indent=2) + "\n" + + +def release_exclusions_sha256() -> str: + return hashlib.sha256( + exclusions_text( + build_release_exclusions(base_exclusion_record(), load_spec()) + ).encode() + ).hexdigest() + + +def install_exclusions(args) -> None: + """Write the release's exclusion record into the stage and rebind it. + + Prepare staged release 20261010's record (pinned); this step replaces it + in the stage's scoring source and bundle with build_release_exclusions' + bytes and records the change in stage.json, so every later step's input + check binds the installed record. Judge prompts do not render exclusions. + """ + stage = args.stage_dir + receipt_path = stage / "stage.json" + receipt = json.loads(receipt_path.read_text()) + bound = f"publish/{RUN_NAME}/us/reference_exclusions.json" + require(bound in receipt["files"], "stage.json does not bind the exclusions") + if receipt.get("references_installed"): + # With an engine upgrade installed the release's exclusion record is + # the build's, never build_release_exclusions(): re-gate the installed + # build (upgrade_exclusion_records) and bind its record again. + upgrade = verify_installed_references(stage, receipt) + bind_build_exclusions(stage, receipt, upgrade) + write_json(receipt_path, receipt) + print( + f"Installed the {upgrade.engine_version} build's {upgrade.records} " + f"exclusions: {upgrade.sha256[EXCLUSIONS_NAME]}" + ) + return + text = exclusions_text( + build_release_exclusions(base_exclusion_record(), load_spec()) + ) + installed = hashlib.sha256(text.encode()).hexdigest() + previous = receipt.get("exclusions_installed") + if previous is None: + require( + receipt["files"][bound] + == BASE_REFERENCE_SHA256["reference_exclusions.json"], + "the stage's exclusion record is not release 20261010's", + ) + for target in (stage / "scoring", stage / "publish" / RUN_NAME / "us"): + (target / "reference_exclusions.json").write_text(text) + receipt["files"][bound] = installed + receipt["exclusions_installed"] = { + "base_sha256": BASE_REFERENCE_SHA256["reference_exclusions.json"], + "sha256": installed, + "spec_sha256": digest(ROOT / SPEC_PATH), + "records": RELEASE_EXCLUSIONS, + } + write_json(receipt_path, receipt) + print( + f"Installed {RELEASE_EXCLUSIONS} exclusions ({NEW_EXCLUSIONS} new): {installed}" + ) + + +def exclusion_entry( + base: dict | None, + item: dict, + record: dict, + case_dir: Path, + adjudicated_on: str | None = None, +) -> dict: + """The adjudication of one ruled output (or, with ``adjudicated_on`` its + record's day, one output an engine upgrade newly excludes). + + A new entry carries the case's bound Opus 5.5 verdict (in this release's + audit tree) as its judge fields, in release 20261010's entry order. An + output that already has a decision (scenario_051's 2026-09-22 + regeneration) keeps its judge fields, which the restate script set, and + takes the ruling's decision fields in their places; its reasoning keeps the + earlier decision's history ahead of the ruling's text. + """ + from date_adds0928_judge_verdicts import _judge + from release_20261006 import ENTRY_FIELDS, bound_verdict + + source, verdict_kind = EXCLUSION_REASON_CODES[record["reason_code"]] + basis = { + "engine_defect": record.get("law"), + "unlisted_input": record.get("unlisted_input"), + "later_law": record.get("published"), + }[verdict_kind] + decision = { + "adjudicated_failure_source": source, + "adjudicated_failure_subtype": item["adjudicated_failure_subtype"], + "adjudicated_on": adjudicated_on or load_spec()["decided_on"], + "adjudicator": "developer", + "excluded_from_scoring": True, + "reference_verdict": verdict_kind, + "reference_basis": basis, + } + require( + item["adjudicated_failure_source"] == source + and item["reference_verdict"] == verdict_kind, + f"{spec_key(item)}: the spec's classes do not match the record's reason", + ) + if base is None: + verdict, meta = bound_verdict(case_dir) + entry = { + "country": "us", + "scenario_id": item["scenario_id"], + "variable": item["variable"], + "judge_model": _judge(meta), + "judge_failure_source": verdict["case_failure_source"], + "judge_failure_subtype": verdict["case_failure_subtype"], + **decision, + "judge_reference_suspect": bool(verdict.get("reference_suspect")), + "reasoning": item["reasoning"], + "judged_on_utc": meta["judged_at_utc"][:10], + } + entry = {name: entry[name] for name in ENTRY_FIELDS} + require(list(entry) == list(ENTRY_FIELDS), "entry fields out of order") + return entry + entry = {} + for name, value in base.items(): + entry[name] = decision.get(name, value) + if name == "adjudicator" and "excluded_from_scoring" not in base: + # In release 20261010's entry order, the flag follows the adjudicator. + entry["excluded_from_scoring"] = True + require( + set(decision) <= set(entry), f"{spec_key(item)}: a decision field is missing" + ) + entry["reasoning"] = base["reasoning"] + " " + item["reasoning"] + return entry + + +# A triage decision's fields, in release 20261010's entry order (an affirmed +# reference carries no excluded_from_scoring, as release 20261010's do not). +TRIAGE_FIELDS = ( + "country", + "scenario_id", + "variable", + "judge_model", + "judge_failure_source", + "judge_failure_subtype", + "adjudicated_failure_source", + "adjudicated_failure_subtype", + "adjudicated_on", + "adjudicator", + "judge_reference_suspect", + "reference_verdict", + "reference_basis", + "reasoning", + "judged_on_utc", +) +TRIAGE_ITEM_KEYS = frozenset( + { + "scenario_id", + "variable", + "adjudicated_failure_source", + "adjudicated_failure_subtype", + "adjudicated_on", + "reference_verdict", + "reference_basis", + "reasoning", + } +) + + +def triage_items(upgrade, spec: dict | None = None) -> list[dict]: + """docs/musewonder/spec.json's triage_adjudications: the developer's decisions + on re-opened cases whose verdict triage could not accept as it stands (a + reference flag the decision answers). Each affirms the reference: the + output stays scored and the case's rows take the decided class. They + answer the upgraded references' verdicts, so they are read only with an + engine upgrade installed; without one, none.""" + if upgrade is None: + return [] + spec = load_spec() if spec is None else spec + items = spec.get("triage_adjudications", []) + require( + isinstance(items, list) + and all(isinstance(i, dict) and set(i) == TRIAGE_ITEM_KEYS for i in items), + "the spec's triage_adjudications must be items with " + f"{sorted(TRIAGE_ITEM_KEYS)}", + ) + keys = [spec_key(item) for item in items] + require(len(set(keys)) == len(keys), f"a triage decision is listed twice: {keys}") + require( + all(item["reference_verdict"] == "affirmed" for item in items), + "a triage decision may only affirm a reference; an exclusion is ruled", + ) + return sorted(items, key=spec_key) + + +def triage_entry(item: dict, case_dir: Path) -> dict: + """The adjudication of a triage decision, with the case's bound Opus 5.5 + verdict as its judge fields.""" + from date_adds0928_judge_verdicts import _judge + from release_20261006 import bound_verdict + + verdict, meta = bound_verdict(case_dir) + entry = { + "country": "us", + "scenario_id": item["scenario_id"], + "variable": item["variable"], + "judge_model": _judge(meta), + "judge_failure_source": verdict["case_failure_source"], + "judge_failure_subtype": verdict["case_failure_subtype"], + "adjudicated_failure_source": item["adjudicated_failure_source"], + "adjudicated_failure_subtype": item["adjudicated_failure_subtype"], + "adjudicated_on": item["adjudicated_on"], + "adjudicator": "developer", + "judge_reference_suspect": bool(verdict.get("reference_suspect")), + "reference_verdict": item["reference_verdict"], + "reference_basis": item["reference_basis"], + "reasoning": item["reasoning"], + "judged_on_utc": meta["judged_at_utc"][:10], + } + require(list(entry) == list(TRIAGE_FIELDS), "triage entry fields out of order") + return entry + + +def exclusion_adjudications( + record: dict, + cases_dir: Path, + regenerated: frozenset = frozenset(), + added: list[tuple[dict, dict]] | tuple = (), + triage: list[dict] | tuple = (), +) -> dict: + """``record`` (the staged adjudication record) with the ruled outputs + decided: an existing entry is restated in place, a new one appended in the + spec's order, and the date conventions name the 2026-10-06 wave. Each + ``triage`` item (triage_items) is appended last (triage_entry). + + ``regenerated`` names the ruled outputs an installed engine upgrade + regenerated (spec_regenerated); they are scored again, so none is decided. + ``added`` (upgrade_decisions) decides the outputs the upgrade newly + excludes, each appended after the ruled ones and dated by its record; the + date conventions then name that wave too. + """ + import copy + + spec = load_spec() + records = { + spec_key(r): r for r in spec_records(spec) if spec_key(r) not in regenerated + } + items = [ + item for item in spec["adjudications"] if spec_key(item) not in regenerated + ] + require( + set(records) == {spec_key(item) for item in items}, + "the spec's adjudications and records name different outputs", + ) + restated = { + spec_key(item) for item in spec.get("restated_adjudications", []) + } - regenerated + out = copy.deepcopy(record) + by_key = { + spec_key(entry): index for index, entry in enumerate(out["adjudications"]) + } + require( + set(by_key) & set(records) == restated, + "existing decisions on ruled outputs differ from the spec's restated list: " + f"{sorted(set(by_key) & set(records))}", + ) + appended = [] + for item in items: + k = spec_key(item) + case_dir = cases_dir / f"us__{k[0]}__{k[1]}" + if k in by_key: + out["adjudications"][by_key[k]] = exclusion_entry( + out["adjudications"][by_key[k]], item, records[k], case_dir + ) + else: + appended.append(exclusion_entry(None, item, records[k], case_dir)) + for item, upgraded in added: + k = spec_key(item) + require( + k not in by_key, + f"{k}: the engine upgrade newly excludes an output that already has a " + "decision; restating one is not supported", + ) + appended.append( + exclusion_entry( + None, + item, + upgraded, + cases_dir / f"us__{k[0]}__{k[1]}", + adjudicated_on=upgraded["decided_on"], + ) + ) + waves = {upgrade_wave(added)} - {None} + if spec_records(spec): + waves.add(spec["decided_on"]) + for item in triage: + k = spec_key(item) + require( + item["adjudicated_on"] in waves, + f"{k}: a triage decision is dated by a wave this release names " + f"({sorted(waves)}), not {item['adjudicated_on']}", + ) + require( + k not in by_key, + f"{k}: a triage decision on a case that already has a decision; " + "restating one is not supported", + ) + appended.append(triage_entry(item, cases_dir / f"us__{k[0]}__{k[1]}")) + out["adjudications"] = out["adjudications"] + appended + out["date_conventions"] = release_date_conventions( + out["date_conventions"], upgrade_wave(added) + ) + return out + + +def release_date_conventions(conventions: str, wave: str | None = None) -> str: + """Release 20261010's date conventions with each wave this release adds + named; the one definition the adjudicate step writes and the record gate + expects. + + A wave is added for the spec's ruled records, when it rules any (its + decided_on, with its wave_sentence and adjudications_written_on), and for + ``wave``, the UTC day of an installed engine upgrade whose newly excluded + outputs are decided (upgrade_wave), with the day the spec says those + decisions were written (upgrade_adjudications_written_on). With neither, + the conventions are release 20261010's, unchanged. + """ + require( + conventions.count(DATE_CONVENTIONS_BASE) == 1 + and conventions.count(DATE_CONVENTIONS_ANCHOR) == 1, + "the record's date conventions are not release 20261010's", + ) + spec = load_spec() + days, sentence = [], "" + if spec_records(spec): + day = spec["decided_on"] + days.append(day) + sentence += ( + f" The {day} wave's decisions {spec['wave_sentence']} and were written " + f"on {spec['adjudications_written_on']} UTC." + ) + if wave is not None: + upgrade_written = spec.get("upgrade_adjudications_written_on") + require( + isinstance(upgrade_written, str) and upgrade_written.strip(), + f"{SPEC_PATH} names no upgrade_adjudications_written_on, the UTC day " + "the engine upgrade's decisions were written", + ) + require(wave not in days, f"the {wave} wave is named twice") + days.append(wave) + sentence += ( + f" The {wave} wave's decisions exclude the outputs the references' " + "second engine upgrade moved onto an input the prompt does not state, " + f"and were written on {upgrade_written} UTC." + ) + if not days: + return conventions + require( + DATE_CONVENTIONS_BASE.count(DATE_CONVENTIONS_LAST_WAVE) == 1, + "the base conventions do not end their wave list on 2026-10-10", + ) + named = ", ".join(["2026-10-10", *days[:-1]]) + f" or {days[-1]})" + waves = DATE_CONVENTIONS_BASE.replace(DATE_CONVENTIONS_LAST_WAVE, ", " + named) + conventions = conventions.replace(DATE_CONVENTIONS_BASE, waves) + return conventions.replace( + DATE_CONVENTIONS_ANCHOR, DATE_CONVENTIONS_ANCHOR + sentence + ) + + +def upgrade_decisions(upgrade, spec: dict | None = None) -> list[tuple[dict, dict]]: + """The (spec item, build record) pairs that decide the outputs an installed + engine upgrade newly excludes, in (scenario_id, variable) order. + + docs/musewonder/spec.json's upgrade_adjudications holds one item per such + output, in the spec adjudications' shape (classes and reasoning); each + record is the build's, computed on the build's engine and decided on its + revision's day. Without an upgrade, or one that excludes nothing new, none; + the spec may then name none either. + """ + spec = load_spec() if spec is None else spec + items = spec.get("upgrade_adjudications", []) + require( + isinstance(items, list) and all(isinstance(i, dict) for i in items), + "the spec's upgrade_adjudications is not a list of items", + ) + if upgrade is None: + return [] + added = upgrade.added + keys = [spec_key(item) for item in items] + if not added: + require( + not keys, + f"the spec's upgrade_adjudications names {sorted(keys)}, but the engine " + "upgrade newly excludes nothing", + ) + return [] + require( + len(set(keys)) == len(keys) and set(keys) == set(added), + "the spec's upgrade_adjudications must decide exactly the outputs the " + f"engine upgrade newly excludes: {sorted(added)}, not {sorted(keys)}", + ) + records = { + spec_key(record): record + for record in json.loads(upgrade.exclusions_text)["exclusions"] + } + pairs = [] + for item in sorted(items, key=spec_key): + record = records[spec_key(item)] + require( + record["engine_version"] == upgrade.engine_version + and record["decided_on"] == upgrade.revision["date"], + f"{spec_key(item)}: the build's record is not decided on its engine and " + "day", + ) + pairs.append((item, record)) + return pairs + + +def upgrade_wave(added) -> str | None: + """The day the upgrade's newly excluded outputs were decided, if any.""" + days = {record["decided_on"] for _, record in added} + require(len(days) <= 1, f"the upgrade's new records name several days: {days}") + return next(iter(days), None) + + +def adjudicate_exclusions(args) -> None: + """Decide the ruled outputs in the staged record (after judge and restate). + + With an engine upgrade installed it decides only the ruled outputs the + build keeps excluded, and drops the decisions of the release 20261010 + records the build regenerated, as release 20260922c dropped scenario_045 + SNAP's (#178). Each drop is recorded in stage.json (adjudications_dropped) + before the record is written, so triage can hold the record to it. + """ + from policybench.adjudications import parse_adjudications + + path = args.stage_dir / "publish" / RUN_NAME / "annotations" / ADJUDICATIONS + record = json.loads(path.read_text()) + upgrade = stage_upgrade(args.stage_dir) + regenerated = frozenset() if upgrade is None else upgrade.regenerated_ruled + added = upgrade_decisions(upgrade) + keys = {spec_key(item) for item in load_spec()["adjudications"]} + keys |= {spec_key(item) for item, _ in added} + require( + not any( + entry.get("excluded_from_scoring") and spec_key(entry) in keys - regenerated + for entry in record["adjudications"] + ), + "the ruled outputs are already decided in the staged record", + ) + require( + not any( + entry.get("excluded_from_scoring") and spec_key(entry) in regenerated + for entry in record["adjudications"] + ), + "the staged record excludes ruled outputs the engine upgrade regenerated; " + "put the staged record back to release 20261010's (from git at " + "BASE_COMMIT), then restate and decide again", + ) + # Drop the regenerated records' decisions first, so a triage decision may + # affirm a regenerated reference on the same case. + dropped: list[dict] = [] + if upgrade is not None: + record, dropped = drop_regenerated_adjudications(record, upgrade) + staged = exclusion_adjudications( + record, + args.stage_dir / "audit" / "cases", + regenerated, + added, + triage_items(upgrade), + ) + parse_adjudications(staged, path) + if upgrade is not None: + receipt_path = args.stage_dir / "stage.json" + receipt = json.loads(receipt_path.read_text()) + receipt["adjudications_dropped"] = dropped + write_json(receipt_path, receipt) + path.write_text(record_text(staged)) + decided = len(keys - regenerated) + print( + f"Decided {decided} ruled outputs ({len(added)} the upgrade excludes) in " + f"{path}" + + (f"; dropped {len(dropped)} regenerated decisions" if upgrade else "") + ) + + +def record_text(record: dict) -> str: + """An adjudication record as the committed file spells it.""" + return json.dumps(record, indent=2, ensure_ascii=False) + "\n" + + +def verify_record_form(text: str, base: dict, wave: str | None = None) -> None: + """The staged record's bytes are exactly its parsed content, and its top + level other than the entries is release 20261010's. + + Duplicate keys, or any other bytes a parser drops, would ship in the + frozen file unseen by the entry gate; so would a rewritten note or date + convention. + """ + record = json.loads(text) + require( + text == record_text(record), + "the staged adjudication record is not in the committed form (duplicate " + "keys or other bytes the entry gate cannot see); write it with " + "json.dumps(indent=2, ensure_ascii=False)", + ) + expected = {k: v for k, v in base.items() if k != "adjudications"} + expected["date_conventions"] = release_date_conventions( + expected["date_conventions"], wave + ) + require( + record_text({k: v for k, v in record.items() if k != "adjudications"}) + == record_text(expected), + "the staged adjudication record changes its note, schema or date " + "conventions beyond naming this release's waves; only entries may change", + ) + + +def verify_restatements( + base: list[dict], staged: list[dict], rejudged: frozenset[str], cases_dir: Path +) -> None: + """A re-opened entry's judge fields must be the restate script's. + + Where they differ from 20261010's, the entry must name the case's current + Opus 5.5 verdict (bound by its sidecar) as its judge, date it by that + sidecar's UTC day, and keep 20261010's judge_previous with exactly one + item appended: the replaced verdict, which is the one 20261010's entry + names (its judge, classes, flag and day are its seed verdict's, as the + sha256-bound sidecar records it). A new entry must name the current judge + too. + """ + from restate_gpt61sol_adjudications import JUDGE_FIELDS, _utc_day, named_item + + before = {case_id(entry): entry for entry in base} + wrong = [] + for entry in staged: + case = case_id(entry) + if case not in rejudged: + continue + original = before.get(case) + judge = {k: v for k, v in entry.items() if k in JUDGE_FIELDS} + if original is not None and judge == { + k: v for k, v in original.items() if k in JUDGE_FIELDS + }: + continue + verdict = cases_dir / case / "verdict.json" + meta_path = verdict.with_name("verdict.meta.json") + meta = json.loads(meta_path.read_text()) if meta_path.is_file() else {} + bound = verdict.is_file() and meta.get("verdict_sha256") == digest(verdict) + day = _utc_day(meta["judged_at_utc"]) if meta.get("judged_at_utc") else None + problems = [] + if not bound or entry.get("judge_model") != JUDGE_MODEL: + problems.append("judge is not the current Opus 5.5 verdict") + if original is not None: + previous = original.get("judge_previous", []) + restated = entry.get("judge_previous", []) + if len(restated) != len(previous) + 1 or restated[:-1] != previous: + problems.append("judge_previous is not 20261010's plus one item") + elif restated[-1] != named_item(original): + problems.append( + "the appended judge_previous item is not the verdict " + "20261010's entry names" + ) + if ( + entry.get("judge_rejudged_on") != day + or entry.get("judged_on_utc", day) != day + ): + problems.append(f"not dated by the current verdict's day {day}") + if problems: + wrong.append(f"{case}: {'; '.join(problems)}") + require( + not wrong, + f"re-opened adjudications not restated by the restate script: {wrong[:4]}", + ) + + +def rejudged_cases(stage: Path) -> frozenset[str]: + """The cases Muse Wonder re-opened: prompt-changes.json's changed and added. + + load_seed re-derives prompt-changes.json from the stage's prompts and the + seed stage.json binds, so the lists cannot drift from the stage. + """ + load_seed(stage) + changes = json.loads((stage / PROMPT_CHANGES).read_text()) + return frozenset(changes["changed"]) | frozenset(changes["added"]) + + +def case_id(entry: dict) -> str: + """An adjudication entry's audit case id.""" + return f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + + +# The only published wording an amendment may change, and where it lives: the +# decision's reasoning in the adjudication record, the case note, and one +# model's row annotation. None of them carries a class, an exclusion or a score. +AMENDABLE_FIELDS = { + "reasoning": "adjudication record", + "case_annotation": "case note", + "annotation": "row annotation", +} + + +def load_amendments(stage: Path, rejudged: frozenset[str]) -> list[dict]: + """The stage's wording-only amendments, each checked for shape and scope. + + Each names a case Muse Wonder re-opened, a wording field, the exact old + text, the new text and the reason; a row annotation also names its model. + An absent file lists none. + """ + path = stage / AMENDMENTS + if not path.exists(): + return [] + payload = json.loads(path.read_text()) + amendments = payload.get("amendments") if isinstance(payload, dict) else None + require(isinstance(amendments, list), f"{AMENDMENTS}: 'amendments' is not a list") + for item in amendments: + require(isinstance(item, dict), f"{AMENDMENTS}: {item!r} is not an object") + field = item.get("field") + require( + field in AMENDABLE_FIELDS, + f"{AMENDMENTS}: field {field!r} is not wording; only " + f"{sorted(AMENDABLE_FIELDS)} may be amended", + ) + keys = {"case_id", "field", "old", "new", "reason"} + if field == "annotation": + keys.add("model") + require( + set(item) == keys, + f"{AMENDMENTS}: an amendment of {field} has keys {sorted(keys)}, " + f"not {sorted(item)}", + ) + require( + item["case_id"] in rejudged, + f"{AMENDMENTS}: {item['case_id']} was not re-judged in this stage", + ) + for key in ("old", "new", "reason", *(("model",) if "model" in keys else ())): + require( + isinstance(item[key], str) and item[key].strip(), + f"{AMENDMENTS}: {key} must be non-empty text: {item!r}", + ) + require( + item["old"] != item["new"], + f"{AMENDMENTS}: an amendment changes nothing: {item!r}", + ) + return amendments + + +def amend_text(text: str, amendments: list[dict], label: str) -> str: + """``text`` with each amendment's old wording, found exactly once, replaced.""" + for item in amendments: + count = text.count(item["old"]) + require( + count == 1, + f"{label}: the old text of a wording amendment occurs {count} times, " + f"not once: {item['old'][:80]!r}", + ) + text = text.replace(item["old"], item["new"]) + return text + + +def _record_amendments(amendments: list[dict]) -> dict[str, list[dict]]: + """The amendments of the adjudication record, by case.""" + grouped: dict[str, list[dict]] = {} + for item in amendments: + if item["field"] == "reasoning": + grouped.setdefault(item["case_id"], []).append(item) + return grouped + + +def ruled_adjudication_problems( + before: dict[str, dict], + after: dict[str, dict], + cases_dir: Path, + regenerated: frozenset = frozenset(), + added: list[tuple[dict, dict]] | tuple = (), + triage: list[dict] | tuple = (), +) -> list[str]: + """Where the staged decisions on the ten ruled outputs are not exactly what + exclusion_entry builds: a new one from its case's bound verdict, and + scenario_051's from its release 20261010 entry with the judge fields the + restate script gave it. + + A ruled output an installed engine upgrade regenerated (``regenerated``) + is scored again: no staged decision may exclude it, and any other decision + on it is an ordinary re-opened case's (verify_adjudication_changes). Each + output the upgrade newly excludes (``added``, upgrade_decisions) must have + exactly the new entry exclusion_entry builds from its item, its build + record and its case's bound verdict, dated by the record.""" + from restate_gpt61sol_adjudications import JUDGE_FIELDS + + spec = load_spec() + records = {spec_key(r): r for r in spec_records(spec)} + problems = [] + for item in spec["adjudications"]: + k = spec_key(item) + case = f"us__{k[0]}__{k[1]}" + if k in regenerated: + if after.get(case, {}).get("excluded_from_scoring"): + problems.append( + f"{case}: the engine upgrade regenerated this output, so no " + "decision may exclude it" + ) + continue + if case not in after: + problems.append(f"{case}: no staged decision") + continue + base = before.get(case) + if base is not None: + # The staged entry before the ruling: release 20261010's decision + # fields and reasoning, with the judge fields the restate script + # wrote. Outside the judge fields it must be 20261010's entry. + staged = after[case] + pre = { + name: value if name in JUDGE_FIELDS else base[name] + for name, value in staged.items() + if name in base or name in JUDGE_FIELDS + } + + def outside_judge(entry: dict) -> list: + return [[n, v] for n, v in entry.items() if n not in JUDGE_FIELDS] + + if outside_judge(pre) != outside_judge(base): + problems.append(f"{case}: fields moved outside the ruling") + continue + base = pre + expected = exclusion_entry(base, item, records[k], cases_dir / case) + if json.dumps(after[case], ensure_ascii=False) != json.dumps( + expected, ensure_ascii=False + ): + problems.append(f"{case}: differs from the ruling's entry") + for item, record in added: + k = spec_key(item) + case = f"us__{k[0]}__{k[1]}" + if case in before: + problems.append(f"{case}: release 20261010 already decides it") + continue + if case not in after: + problems.append(f"{case}: no staged decision") + continue + expected = exclusion_entry( + None, item, record, cases_dir / case, adjudicated_on=record["decided_on"] + ) + if json.dumps(after[case], ensure_ascii=False) != json.dumps( + expected, ensure_ascii=False + ): + problems.append(f"{case}: differs from the upgrade's entry") + for item in triage: + k = spec_key(item) + case = f"us__{k[0]}__{k[1]}" + if case not in after: + problems.append(f"{case}: no staged triage decision") + continue + expected = triage_entry(item, cases_dir / case) + if json.dumps(after[case], ensure_ascii=False) != json.dumps( + expected, ensure_ascii=False + ): + problems.append(f"{case}: differs from the triage decision's entry") + return problems + + +def verify_adjudication_changes( + base: list[dict], + staged: list[dict], + rejudged: frozenset[str], + amendments: list[dict], + cases_dir: Path, + *, + regenerated: frozenset = frozenset(), + dropped: frozenset[str] = frozenset(), + added: list[tuple[dict, dict]] | tuple = (), + triage: list[dict] | tuple = (), +) -> int: + """A staged record differs from 20261010's only where it has a reason to. + + Only a case Muse Wonder re-opened (``rejudged``) may change, and only in + its judge fields (the restate script's JUDGE_FIELDS) and in the reasoning + wording the listed amendments change, exactly as they say. Every other + field of every committed entry keeps its value and its place, key order + included, and the committed entries keep their order. A new entry may only + decide a re-opened case, and none may be dropped. Returns how many staged + entries are new. + + With an engine upgrade installed, ``regenerated`` names the ruled outputs + it regenerated (no longer decided as ruled) and ``dropped`` the cases of + the release 20261010 records it regenerated: exactly those decisions must + be gone, and every other one stays. ``added`` (upgrade_decisions) decides + the outputs it newly excludes, gated as the ruled outputs are. + """ + from restate_gpt61sol_adjudications import JUDGE_FIELDS + + def serialized(entry: dict, rejudged_case: bool) -> str: + # A re-opened case may rewrite its judge fields; nothing else may move. + items = [ + [key, value] + for key, value in entry.items() + if not (rejudged_case and key in JUDGE_FIELDS) + ] + return json.dumps(items, ensure_ascii=False, allow_nan=False) + + before = {case_id(entry): entry for entry in base} + after = {case_id(entry): entry for entry in staged} + ruled = ruled_adjudication_problems( + before, after, cases_dir, regenerated, added, triage + ) + triage_cases = {f"us__{k[0]}__{k[1]}" for k in map(spec_key, triage)} + require( + triage_cases <= rejudged, + "triage decisions name cases Muse Wonder's stage did not re-open: " + f"{sorted(triage_cases - rejudged)[:8]}", + ) + require( + not (triage_cases & set(before)) or triage_cases & set(before) <= dropped, + "a triage decision replaces a recorded decision: " + f"{sorted((triage_cases & set(before)) - dropped)[:8]}", + ) + require(not ruled, f"Staged adjudications of the ruled outputs: {ruled[:8]}") + ruled_cases = { + f"us__{k[0]}__{k[1]}" + for k in (spec_key(item) for item in load_spec()["adjudications"]) + if k not in regenerated + } | {f"us__{k[0]}__{k[1]}" for k in (spec_key(item) for item, _ in added)} + ruled_cases |= triage_cases + stray = sorted(dropped - set(before)) + require( + not stray, + f"Staged adjudications drop decisions release 20261010 lacks: {stray[:8]}", + ) + # A regenerated record's decision is gone; a triage decision may affirm the + # regenerated reference in its place (it is in ruled_cases, gated above). + kept = sorted((dropped & set(after)) - triage_cases) + require( + not kept, + "Staged adjudications keep the decisions of records the engine upgrade " + f"regenerated; run --step adjudicate-exclusions: {kept[:8]}", + ) + before = { + case: entry + for case, entry in before.items() + if case not in ruled_cases and case not in dropped + } + after = {case: entry for case, entry in after.items() if case not in ruled_cases} + dropped = sorted(set(before) - set(after)) + require(not dropped, f"Staged adjudications drop recorded decisions: {dropped}") + new = sorted(set(after) - set(before)) + require( + set(new) <= rejudged, + "Staged adjudications add decisions on cases Muse Wonder did not " + f"re-open: {sorted(set(new) - rejudged)[:8]}", + ) + require( + [case for case in after if case in before] == list(before), + "Staged adjudications change the committed entry order", + ) + grouped = _record_amendments(amendments) + require( + set(grouped) <= set(before), + f"Wording amendments name cases with no recorded decision: " + f"{sorted(set(grouped) - set(before))}", + ) + changed = [] + for case, entry in before.items(): + if case in grouped: + entry = { + **entry, + "reasoning": amend_text( + entry["reasoning"], grouped[case], f"{case} reasoning" + ), + } + reopened = case in rejudged + if serialized(after[case], reopened) != serialized(entry, reopened): + changed.append(case) + require( + not changed, + "Staged adjudications change recorded decisions beyond the re-judged " + f"cases' judge fields and the listed wording amendments: {changed[:8]}", + ) + return len(new) + + +def stage_adjudications( + path: Path, + rejudged: frozenset[str], + amendments: list[dict], + cases_dir: Path, + *, + regenerated: frozenset = frozenset(), + dropped: frozenset[str] = frozenset(), + added: list[tuple[dict, dict]] | tuple = (), + triage: list[dict] | tuple = (), +) -> list[dict]: + """The staged record, with its listed reasoning amendments applied. + + Checked in memory against release 20261010's record (from git) and the + stage's verdicts; written back only when an amendment was not applied yet. + ``regenerated``, ``dropped`` and ``added`` describe an installed engine + upgrade (verify_adjudication_changes). + """ + from freeze_snapshot import verify_adjudications_keep_judge_verdicts + + from policybench.adjudications import parse_adjudications + + base_record = base_adjudication_record() + base = base_adjudications() + current = path.read_text() + wave = upgrade_wave(added) + verify_record_form(current, base_record, wave) + record = json.loads(current) + grouped = _record_amendments(amendments) + original = {case_id(entry): entry for entry in base} + applied = False + for entry in record["adjudications"]: + case = case_id(entry) + if case in grouped and case in original: + wording = original[case]["reasoning"] + if entry["reasoning"] == wording: + entry["reasoning"] = amend_text( + wording, grouped[case], f"{case} reasoning" + ) + applied = True + text = record_text(record) + verify_record_form(text, base_record, wave) + entries = parse_adjudications(json.loads(text), path) + verify_adjudication_changes( + base, + entries, + rejudged, + amendments, + cases_dir, + regenerated=regenerated, + dropped=dropped, + added=added, + triage=triage, + ) + # A dropped decision is gone, so an entry on its case is new, not restated. + verify_restatements( + [entry for entry in base if case_id(entry) not in dropped], + entries, + rejudged, + cases_dir, + ) + verify_adjudications_keep_judge_verdicts(entries, cases_dir) + if applied: + pending = path.with_name(path.name + ".amending") + pending.write_text(text) + os.replace(pending, path) + return entries + + +def amend_annotations(rows, cases, amendments: list[dict]) -> None: + """Apply the listed case-note and row-annotation amendments in place. + + Each must find exactly one row, and its old text exactly once in it. + """ + for item in amendments: + if item["field"] == "reasoning": + continue + country, scenario, variable = item["case_id"].split("__", 2) + frame = cases if item["field"] == "case_annotation" else rows + mask = ( + (frame["country"].astype(str) == country) + & (frame["scenario_id"].astype(str) == scenario) + & (frame["variable"].astype(str) == variable) + ) + if item["field"] == "annotation": + mask &= frame["model"].astype(str) == item["model"] + require( + int(mask.sum()) == 1, + f"a wording amendment of {item['case_id']} {item['field']} " + f"matches {int(mask.sum())} rows, not one", + ) + index = frame.index[mask][0] + frame.loc[index, item["field"]] = amend_text( + str(frame.loc[index, item["field"]]), + [item], + f"{item['case_id']} {item['field']}", + ) + + +def annotation_csv_text(frame) -> str: + """The bytes triage writes for the row annotations or the case notes, and + the freeze rebuilds: every column, in collect_audit's row order.""" + return frame.to_csv(index=False) + + +def resolve_live_base(args) -> dict: + """The 20261010 payload an export compares the incumbents against. + + Before the freeze the live pointer and the committed snapshot are + 20261010's, and resolve_base checks all of them. After the freeze they are + this release's own, so a re-export (after a fix to the staged annotations) + reads the 20261010 run payload from BASE_COMMIT instead and checks that it + rewraps to the 20261010 release asset (BASE_SHA256). Any other pointer is + refused. + """ + pointer = live_pointer() + if pointer["tag"] == BASE_TAG: + return resolve_base(args)[2] + require( + pointer["tag"] == RELEASE_TAG, + "base pointer changed; review the base before staging", + ) + return base_payload_from_commit() + + +def prepare_inputs(args, runs, base, reference) -> tuple[Path, dict]: + """Copy immutable inputs and fold full or explicitly partial cohorts.""" + import pandas as pd + + from policybench.config import MODELS as registry + from policybench.config import PRICE_OVERRIDES_PER_1M + from policybench.fold_board import fold_board + + stage = args.stage_dir + frames = {} + provenance = {} + for run in runs: + frame = pd.read_csv(run.predictions, low_memory=False) + if args.partial: + frame = frame.drop(columns=["raw_response"], errors="ignore") + households = set(frame.scenario_id) + require( + len(households) == run.state["total"], + f"state/CSV household mismatch: {run.model}", + ) + require( + households <= set(reference.scenario_id), f"unknown households: {run.model}" + ) + validate_keys( + frame, reference[reference.scenario_id.isin(households)], run.model + ) + require( + run.model in registry and run.model in PRICE_OVERRIDES_PER_1M, + f"missing model registration/pricing: {run.model}", + ) + if not args.partial: + require( + run.state["total"] == 100 and households == set(reference.scenario_id), + f"full release requires 100 households: {run.model}", + ) + require( + not run.state.get("synthetic_partial"), + "synthetic run state is partial only", + ) + fingerprint = run.state.get("treatment_fingerprint") + require( + isinstance(fingerprint, dict) + and fingerprint.get("model_id") == registry[run.model], + f"missing/mismatched treatment fingerprint: {run.model}", + ) + import freeze_snapshot as freezer + from freeze_gpt61sol import validate_treatment + + validate_treatment( + freezer, + run.state, + run.run_dir / "run_state.json", + SNAPSHOT / "scenarios.csv", + ) + for field in ( + "estimated_cost_usd", + "total_tokens", + "provider_resolved_model", + "provider_response_id", + ): + require( + field in frame and frame[field].notna().all(), + f"missing {field}: {run.model}", + ) + expected_provider = registry[run.model].split("/")[-1] + require( + set(frame.provider_resolved_model) == {expected_provider}, + f"unexpected provider-resolved model: {run.model}", + ) + target = stage / "inputs" / run.slug + target.mkdir(parents=True, exist_ok=True) + shutil.copyfile(run.predictions, target / "predictions.csv") + shutil.copyfile(run.run_dir / "run_state.json", target / "run_state.json") + frames[run.model] = frame + provenance[run.model] = { + "households": len(households), + "rows": len(frame), + "predictions_sha256": digest(target / "predictions.csv"), + "treatment_fingerprint": run.state.get("treatment_fingerprint"), + "provider_resolved_models": sorted( + frame.provider_resolved_model.dropna().unique() + ), + "provider_system_fingerprints": sorted( + frame.get("provider_system_fingerprint", pd.Series(dtype=str)) + .dropna() + .unique() + ), + } + shared = set.intersection(*(set(frame.scenario_id) for frame in frames.values())) + require(bool(shared), "no households completed by every addition") + selected = ( + reference[reference.scenario_id.isin(shared)] if args.partial else reference + ) + scoring = stage / "scoring" + scoring.mkdir(exist_ok=True) + for name in REFERENCE_FILES: + shutil.copyfile(SNAPSHOT / name, scoring / name) + if args.partial: + selected.to_csv(scoring / "reference_outputs.csv", index=False) + meta = json.loads((scoring / "reference_outputs.csv.meta.json").read_text()) + meta.update( + reference_csv_sha256=digest(scoring / "reference_outputs.csv"), + row_count=len(selected), + partial=True, + ) + write_json(scoring / "reference_outputs.csv.meta.json", meta) + scenarios = pd.read_csv(scoring / "scenarios.csv") + scenarios[scenarios.scenario_id.isin(shared)].to_csv( + scoring / "scenarios.csv", index=False + ) + exclusions = json.loads((scoring / "reference_exclusions.json").read_text()) + exclusions["exclusions"] = [ + e for e in exclusions["exclusions"] if e["scenario_id"] in shared + ] + write_json(scoring / "reference_exclusions.json", exclusions) + base = base[base.scenario_id.isin(shared)] + else: + verify_reference_pins(scoring, "staged reference") + base_path = stage / "base-predictions.csv" + base.to_csv(base_path, index=False) + additions = [] + for run in runs: + path = stage / "inputs" / run.slug / "fold.csv" + frames[run.model][frames[run.model].scenario_id.isin(shared)].to_csv( + path, index=False + ) + additions.append(path) + result = fold_board(base_path, additions, scoring, stage / "fold", export=False) + require( + not result["excluded"] and result["models"] == BOARD_MODELS, + f"fold refused additions: {result}", + ) + bundle = stage / "publish" / RUN_NAME + us = bundle / "us" + us.mkdir(parents=True, exist_ok=True) + require( + not (us / "by_model").exists(), "by_model would shadow combined predictions" + ) + shutil.copyfile(stage / "fold/us/predictions.csv", us / "predictions.csv") + for name in REFERENCE_FILES: + shutil.copyfile(scoring / name, us / name) + annotations = bundle / "annotations" + annotations.mkdir(exist_ok=True) + for name in ANNOTATION_FILES: + # Re-preparing must not overwrite a recorded stage adjudication. + if not (annotations / name).exists(): + shutil.copyfile(ANNOTATIONS / name, annotations / name) + write_json(stage / "model-provenance.json", provenance) + return bundle, frames + + +def partial_scores(args, base, reference, frames) -> None: + """Rank each addition against incumbents on its own completed households.""" + import pandas as pd + + from policybench.analysis import analyze_no_tools + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + scored, _ = split_reference(reference, load_reference_exclusions(SNAPSHOT)) + scenarios = pd.read_csv(SNAPSHOT / "scenarios.csv") + results = [] + for model, frame in frames.items(): + ids = set(frame.scenario_id) + cohort = pd.concat([base[base.scenario_id.isin(ids)], frame], ignore_index=True) + analysis = analyze_no_tools( + scored[scored.scenario_id.isin(ids)], + cohort, + scenarios=scenarios[scenarios.scenario_id.isin(ids)], + ) + stats = analysis["bounded_summary"].set_index("model") + exact = float(stats.loc[model, "weighted_exact"]) + rank = 1 + int((stats.weighted_exact > exact).sum()) + result = { + "label": "PARTIAL", + "model": model, + "households": len(ids), + "scored_outputs": int(scored.scenario_id.isin(ids).sum()), + "exact_percent": exact * 100, + "rank": rank, + "models": len(stats), + } + results.append(result) + print(json.dumps(result)) + write_json(args.stage_dir / "partial-scores.json", results) + + +def seed_prompt_digests(audit: Path) -> dict[str, str]: + """The sha256 of every prompt a case carries, keyed by case id.""" + return { + case.name: digest(case / "prompt.md") + for case in sorted((audit / "cases").glob("*")) + if (case / "prompt.md").is_file() + } + + +def seed_digest(audit: Path) -> dict[str, dict[str, str]]: + """Each judged case's prompt and verdict sha256, keyed by case id.""" + seed = {} + for case in sorted((audit / "cases").glob("*")): + if not (case / "prompt.md").is_file(): + continue + require((case / "verdict.json").is_file(), f"seed case unjudged: {case.name}") + seed[case.name] = { + "prompt_sha256": digest(case / "prompt.md"), + "verdict_sha256": digest(case / "verdict.json"), + } + return seed + + +def seed_digest_text(seed: dict[str, dict[str, str]]) -> str: + """The seed digest as docs/musewonder/seed_digest.csv spells it.""" + rows = [ + f"{case},{item['prompt_sha256']},{item['verdict_sha256']}\n" + for case, item in sorted(seed.items()) + ] + return "case_id,prompt_sha256,verdict_sha256\n" + "".join(rows) + + +def verify_seed(seed: dict[str, dict[str, str]]) -> None: + """The seed must be the 20261010 audit the committed digest records.""" + require( + hashlib.sha256(seed_digest_text(seed).encode()).hexdigest() + == SEED_DIGEST_SHA256, + "the audit seed is not release 20261010's (see docs/musewonder/" + "seed_digest.csv): its cases, prompts or verdicts differ", + ) + + +def load_seed(stage: Path) -> dict[str, dict[str, str]]: + """The seed stage.json binds, checked against the committed digest. + + It also re-derives which cases are kept, changed and added from the + stage's prompts and that seed, and refuses unless prompt-changes.json + says the same: a kept case's prompt that drifts, a case moved between + the lists, or a manifest whose wrong models the staged predictions do not + bear out, stops every step that relies on them. + """ + receipt = json.loads((stage / "stage.json").read_text()) + require( + "seed" in receipt, + "stage.json does not bind the audit seed; run --step bind-seed " + "--audit-seed ", + ) + verify_seed(receipt["seed"]) + verify_prompt_changes(stage, receipt["seed"]) + return receipt["seed"] + + +def verify_prompt_changes(stage: Path, seed: dict[str, dict[str, str]]) -> None: + """prompt-changes.json must be what the stage's prompts say against the seed. + + A case whose prompt is the seed's is kept, one whose prompt differs is + changed and one the seed lacks is added; Muse Wonder must be among the + wrong models of every changed or added case, and no seed case may vanish + (check_prompt_changes, as prepare applies it). With an engine upgrade + installed, a case whose reference value or explanation the installed + revision changed may change or appear without it (upgraded_cases, derived + from the staged files against release 20261010's in git). The staged + predictions must agree (verify_reopened_by_predictions). + """ + derived = check_prompt_changes( + stage / "audit", + {case: item["prompt_sha256"] for case, item in seed.items()}, + upgraded=upgraded_cases(stage), + ) + recorded = json.loads((stage / PROMPT_CHANGES).read_text()) + differ = sorted( + { + case + for key in ("kept", "changed", "added") + for case in set(derived[key]) ^ set(recorded.get(key, [])) + } + ) + require( + not differ, + f"{PROMPT_CHANGES} disagrees with the stage's prompts and the bound " + f"seed on {len(differ)} cases: {differ[:8]}", + ) + verify_reopened_by_predictions(stage, derived) + + +def verify_reopened_by_predictions(stage: Path, derived: dict[str, list]) -> None: + """The manifest's claim that Muse Wonder re-opened a case must hold. + + cases.jsonl is editable, so re-score the staged predictions against the + staged references with wrong_prediction_rows, the rule prepare_audit + used to list each case's wrong models. A changed or added case must list + the new model exactly when its prediction is wrong; any other case it + gets wrong must be parse-failure-only as the manifest records it, so a + kept case needs the new model's prediction right. The staged references + are the installed engine upgrade's when one is installed, so a case the + upgrade re-opened without the new model (its prediction right on the new + reference) passes, and one the upgrade made it miss must list it. + """ + from policybench.audit import _case_id, _load_manifest + from policybench.case_annotations import wrong_prediction_rows + + us = stage / "publish" / RUN_NAME / "us" + wrong = wrong_prediction_rows(us) + manifest = _load_manifest(stage / "audit") + reopened = {*derived["changed"], *derived["added"]} + parse_only = {case for case, row in manifest.items() if row["parse_failure_only"]} + differ = set() + for model in MODELS.values(): + rows = wrong[wrong.model == model] + missed = { + _case_id(us.name, str(scenario), str(variable)) + for scenario, variable in zip(rows.scenario_id, rows.variable) + } + differ |= { + case + for case in reopened + if (model in manifest[case]["wrong_models"]) != (case in missed) + } + differ |= missed - reopened - parse_only + require( + not differ, + f"the audit manifest's wrong models disagree with the staged " + f"predictions of {sorted(MODELS.values())} on {len(differ)} cases: " + f"{sorted(differ)[:8]}", + ) + + +def bind_seed(args) -> None: + """Bind the seed in the stage.json of a stage prepared before prepare did. + + The seed must match the committed digest, and the stage must agree with it + case by case: every kept case keeps the seed's prompt and verdict bytes, + every changed case's prompt differs, and no added case is a seed case. + """ + receipt_path = args.stage_dir / "stage.json" + receipt = json.loads(receipt_path.read_text()) + require("seed" not in receipt, "stage.json already binds its audit seed") + require(args.audit_seed is not None, "bind-seed needs --audit-seed") + seed = seed_digest(args.audit_seed) + verify_seed(seed) + changes = json.loads((args.stage_dir / PROMPT_CHANGES).read_text()) + cases = args.stage_dir / "audit" / "cases" + differ = [ + case + for case in changes["kept"] + if case not in seed + or not (cases / case / "verdict.json").is_file() + or digest(cases / case / "prompt.md") != seed[case]["prompt_sha256"] + or digest(cases / case / "verdict.json") != seed[case]["verdict_sha256"] + ] + differ += [ + case + for case in changes["changed"] + if case not in seed + or digest(cases / case / "prompt.md") == seed[case]["prompt_sha256"] + ] + differ += [case for case in changes["added"] if case in seed] + differ += sorted(set(seed) - {*changes["kept"], *changes["changed"]}) + require( + not differ, + f"the stage disagrees with the seed on {len(differ)} cases: {differ[:8]}", + ) + receipt["seed"] = seed + write_json(receipt_path, receipt) + print(f"Bound the audit seed: {len(seed)} judged cases") + + +def reworded_since_seed() -> frozenset[str]: + """The cases whose committed reference explanation differs between the + seed's release (SEED_RELEASE_COMMIT) and the base (BASE_COMMIT). + + Refuses unless they are exactly REWORDED_SINCE_SEED, so a change to any + other explanation since the seed stops prepare instead of re-opening a + case nobody reviewed. + """ + import io + + import pandas as pd + + def explanations(commit: str) -> dict[tuple[str, str], tuple]: + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{REFERENCE_EXPLANATIONS}"], + capture_output=True, + ) + require( + result.returncode == 0, + f"cannot read {REFERENCE_EXPLANATIONS} at {commit[:12]}: " + f"{result.stderr.decode().strip()}", + ) + frame = pd.read_csv(io.BytesIO(result.stdout), dtype=str, keep_default_na=False) + return { + (row.scenario_id, row.variable): tuple(row) + for row in frame.itertuples(index=False) + } + + before = explanations(SEED_RELEASE_COMMIT) + after = explanations(BASE_COMMIT) + require(set(before) == set(after), "explanation keys changed since the seed") + derived = frozenset( + f"us__{scenario}__{variable}" + for (scenario, variable), row in after.items() + if before[(scenario, variable)] != row + ) + require( + derived == REWORDED_SINCE_SEED, + "reference explanations changed since the seed outside the pinned " + f"set: {sorted(derived ^ REWORDED_SINCE_SEED)}", + ) + return derived + + +def check_prompt_changes( + audit: Path, seeded: dict[str, str], upgraded: frozenset[str] = frozenset() +) -> dict[str, list]: + """Only a case Muse Wonder joins, or one release 20261010 reworded, + may change or appear. + + The seed's prompts re-render byte-identically from release 20260930's + committed snapshot and the pinned grounding, so an incumbent-only case + whose prompt differs (or that is new) means an input changed under the + carried-over verdicts, unless its reference explanation is one release + 20261010 reworded (reworded_since_seed), or its reference value or + explanation is one an installed engine upgrade changed (``upgraded``, + upgraded_cases); that case is re-judged. + """ + manifest = [ + json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() + ] + prompts = { + item["case_id"]: digest(audit / "cases" / item["case_id"] / "prompt.md") + for item in manifest + if not item["parse_failure_only"] + } + return classify_prompt_changes( + manifest, prompts, seeded, reworded_since_seed(), upgraded + ) + + +def classify_prompt_changes( + manifest: list[dict], + prompts: dict[str, str], + seeded: dict[str, str], + reworded: frozenset[str], + upgraded: frozenset[str] = frozenset(), +) -> dict[str, list]: + """check_prompt_changes' rule on a manifest and its prompts' sha256. + + install-references applies it to the audit it is about to write, so it + refuses before it writes anything. + """ + new_models = set(MODELS.values()) + changed, added, kept, offending = [], [], [], [] + for item in manifest: + if item["parse_failure_only"]: + continue + case_id = item["case_id"] + prompt = prompts[case_id] + if seeded.get(case_id) == prompt: + kept.append(case_id) + continue + (changed if case_id in seeded else added).append(case_id) + if ( + not new_models & set(item["wrong_models"]) + and case_id not in reworded + and case_id not in upgraded + ): + offending.append(case_id) + require( + not offending, + "incumbent-only case prompts changed against the seed (grounding, " + f"references or annotations differ): {offending[:8]}", + ) + # Adding a model cannot remove a wrong answer, so every judged seed case + # must still be a case. + vanished = sorted(set(seeded) - {*kept, *changed}) + require(not vanished, f"seed cases vanished from the audit: {vanished[:8]}") + unrendered = sorted(reworded - set(changed)) + require( + not unrendered, + f"reworded cases whose prompts did not change: {unrendered[:8]}", + ) + return {"kept": kept, "changed": changed, "added": added} + + +def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: + """Seed a private audit; changed prompts invalidate copied verdicts. + + Returns the seed digest, which main binds in stage.json. + """ + from policybench.audit import prepare_audit + + require( + args.audit_seed is not None and args.grounding is not None, + "prepare needs --audit-seed and --grounding (read-only sources)", + ) + require( + digest(args.grounding) == GROUNDING_SHA256, + "grounding differs from the one the 20260929 audit was rendered with", + ) + audit = args.stage_dir / "audit" + # Compare against the read-only seed, not the stage copy, so a retried + # prepare cannot mistake its own earlier rendering for the seed. + seed = seed_digest(args.audit_seed) + verify_seed(seed) + seeded = {case: item["prompt_sha256"] for case, item in seed.items()} + if not audit.exists(): + for source in sorted((args.audit_seed / "cases").glob("*")): + if not (source / "prompt.md").is_file(): + continue + target = audit / "cases" / source.name + target.mkdir(parents=True) + for name in ("prompt.md", "verdict.json", "verdict.meta.json", "codex.log"): + if (source / name).is_file(): + shutil.copyfile(source / name, target / name) + lookup = grounding_lookup(args.grounding) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) + changes = check_prompt_changes(audit, seeded) + write_json( + args.stage_dir / "prompt-changes.json", + {key: sorted(value) for key, value in changes.items()}, + ) + pending = validate_verdicts(audit, remove_invalid=True, seed=seed) + write_json(args.stage_dir / "pending.json", pending) + print( + f"Prepared audit: {len(changes['kept'])} prompts unchanged, " + f"{len(changes['changed'])} changed and {len(changes['added'])} new; " + f"{len(pending)} cases need Opus 5.5" + ) + return seed + + +def set_aside(audit: Path, case_id: str, reason: str) -> None: + """Move a case's verdict, sidecar and judge evidence out of the audit. + + They go to /rejected-verdicts/// with the judge's + envelope, log and transcript, so a re-judge never destroys the verdict it + replaces or the record of how it was made. + """ + from datetime import datetime, timezone + + case = audit / "cases" / case_id + names = ( + "verdict.json", + "verdict.meta.json", + "claude.json", + "claude.log", + "claude.transcript.jsonl", + ) + files = [case / name for name in names] + if not any(path.exists() for path in files[:2]): + return + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + target = audit.parent / "rejected-verdicts" / case_id / stamp + target.mkdir(parents=True) + (target / "reason.txt").write_text(reason + "\n") + for path in files: + if path.exists(): + os.replace(path, target / path.name) + + +def validate_verdicts( + audit: Path, + remove_invalid: bool = False, + seed: dict[str, dict[str, str]] | None = None, +) -> list[str]: + """Validate schema, model coverage and each verdict's binding to its bytes. + + Every judged verdict's sidecar must carry the verdict's own sha256. A case + whose prompt is the seed's (``seed``, as stage.json binds it) carries its + seed verdict over: the verdict must be the seed's, byte for byte, and any + prompt_sha256 its sidecar records must match. Every other verdict is new + and must record the sha256 of the prompt it judged. A verdict naming + Muse Wonder also needs bound Opus 5.5 provenance. Returns the pending + cases; ``remove_invalid`` sets their verdicts aside. A carried-over verdict + that fails is refused outright: a re-judge cannot restore it. + """ + import jsonschema + + schema = json.loads((audit / "schema.json").read_text()) + manifest = { + item["case_id"]: item + for item in map(json.loads, (audit / "cases.jsonl").read_text().splitlines()) + } + seed = seed or {} + pending, refused = [], [] + for case_id, item in manifest.items(): + if item["parse_failure_only"]: + continue + path = audit / "cases" / case_id / "verdict.json" + meta_path = path.with_name("verdict.meta.json") + prompt = path.with_name("prompt.md") + carried = ( + case_id in seed + and prompt.is_file() + and digest(prompt) == seed[case_id]["prompt_sha256"] + ) + try: + blob = path.read_bytes() + verdict = json.loads(blob) + jsonschema.validate(verdict, schema) + names = [m["model"] for m in verdict["models"]] + if len(names) != len(set(names)) or set(names) != set(item["wrong_models"]): + raise ValueError("wrong model coverage") + meta = json.loads(meta_path.read_text()) if meta_path.is_file() else {} + if not isinstance(meta, dict): + raise ValueError("verdict provenance is not an object") + if meta.get("verdict_sha256") != hashlib.sha256(blob).hexdigest(): + raise ValueError("the sidecar is not bound to this verdict") + bound = meta.get("prompt_sha256") + if carried: + if hashlib.sha256(blob).hexdigest() != seed[case_id]["verdict_sha256"]: + raise ValueError("a carried-over verdict is not the seed's") + if bound is not None and bound != digest(prompt): + raise ValueError("verdict is bound to a different prompt") + elif bound != digest(prompt): + raise ValueError("a new verdict is not bound to this prompt") + if set(names) & set(MODELS.values()): + if ( + meta.get("judge_model_requested") != JUDGE_MODEL + or meta.get("judge_model_reported") != [JUDGE_MODEL] + or not meta.get("judge_runner") + or not meta.get("judged_at_utc") + ): + raise ValueError("missing or mismatched Opus 5.5 provenance") + except (OSError, ValueError, jsonschema.ValidationError) as error: + if carried: + refused.append(f"{case_id} ({error})") + continue + pending.append(case_id) + if remove_invalid: + set_aside(audit, case_id, f"invalid: {error}") + require( + not refused, + f"{len(refused)} carried-over verdicts differ from the seed's; a stage " + f"input changed after prepare: {refused[:4]}", + ) + return sorted(pending) + + +def account_data(value, at: str, keys: bool = True) -> list[str]: + """Where ``value`` carries an e-mail address or a key naming an account. + + Any key that matches ACCOUNT_KEY (unless ``keys`` is false), and any key + or string that holds an e-mail address (EMAIL_ADDRESS), at any depth; + ``at`` names ``value``. + """ + hits = [] + if isinstance(value, dict): + for key, item in value.items(): + where = f"{at}.{key}" + if keys and ACCOUNT_KEY.search(str(key)): + hits.append(f"key {where}") + if EMAIL_ADDRESS.search(str(key)): + hits.append(f"an e-mail address in key {where}") + hits += account_data(item, where, keys) + elif isinstance(value, list): + for index, item in enumerate(value): + hits += account_data(item, f"{at}[{index}]", keys) + elif isinstance(value, str) and EMAIL_ADDRESS.search(value): + hits.append(f"an e-mail address at {at}") + return hits + + +def outside_the_judge(event: dict, prompt: str | None) -> dict: + """``event`` without the judged prompt's own user message or the judge's + words: the text of a user event that is exactly ``prompt``, and the content + of an assistant turn. The account-data rule reads everything else.""" + message = event.get("message") + if isinstance(message, dict) and ( + event.get("type") == "assistant" + or ( + event.get("type") == "user" + and prompt is not None + and message.get("content") == prompt + ) + ): + rest = {key: value for key, value in message.items() if key != "content"} + return {**event, "message": rest} + return event + + +def transcript_problems( + path: Path, effort: str | None, prompt: bytes, verdict: object +) -> list[str]: + """Why a judge's transcript does not show an isolated judge; [] if it does. + + A Python port of the transcript checks in scripts/run_audit_claude.sh + (extract_verdict): no tool call but StructuredOutput, no event of a type + outside its EVENT_TYPES (JUDGE_EVENT_TYPES), no context attachment outside + its ATTACHMENTS (JUDGE_ATTACHMENTS; so no skill listing and no + credential_org record), a session context that is empty (no account e-mail + or git status), no account data anywhere outside the prompt's own user + message and the judge's own turns (account_data, outside_the_judge), no + working directory inside a git repository, every assistant turn at the + effort the sidecar records, when it records one (the sidecars of the + stage's first 17 isolated verdicts record none), no advisor model, and + user events that are exactly one text message, the case's ``prompt`` + (prompt.md's bytes, decoded as UTF-8), besides Claude Code's own nudge + (JUDGE_PROMPT_NUDGE) and the results of the judge's StructuredOutput calls. + It also requires exactly one accepted StructuredOutput call, whose input + is the published ``verdict`` (verdict.json, parsed): any other call must + be one the schema refused, which the judge then answered again. + """ + try: + lines = path.read_text().splitlines() + except OSError: + return ["no transcript"] + try: + text = prompt.decode("utf-8") + except UnicodeDecodeError: + text = None + problems, calls, refused = [], [], set() + # The user events: text messages (the prompt) and everything else (which + # may only answer the judge's own StructuredOutput calls, by id). + texts, others, answers = [], [], set() + + def answer(part: dict) -> bool: + return part.get("type") == "tool_use" and part.get("name") == "StructuredOutput" + + for number, line in enumerate(lines, 1): + try: + event = json.loads(line) + except ValueError: + problems.append(f"transcript line {number} is not JSON") + continue + if not isinstance(event, dict): + problems.append(f"transcript line {number} is not an event") + continue + if event.get("type") not in JUDGE_EVENT_TYPES: + problems.append(f"an event of type {event.get('type')!r}") + problems += account_data(outside_the_judge(event, text), f"line {number}") + if event.get("type") == "attachment": + attachment = event.get("attachment") or {} + kind = attachment.get("type") + if kind not in JUDGE_ATTACHMENTS: + problems.append(f"a {kind!r} attachment") + if kind == "environment" and (attachment.get("snapshot") or {}).get( + "isGitRepo" + ): + problems.append("a working directory inside a git repository") + context = attachment.get("context") + if kind == "session_context" and context != {}: + carried = sorted(context) if isinstance(context, dict) else context + problems.append(f"a session context carrying {carried!r}") + if event.get("type") == "assistant": + if effort is not None and event.get("effort") != effort: + problems.append(f"a turn at effort {event.get('effort')!r}") + if event.get("advisorModel"): + problems.append(f"advisor model {event.get('advisorModel')!r}") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + if event.get("type") == "user": + if not isinstance(content, str): + others.append(content) + elif not (event.get("isMeta") is True and content == JUDGE_PROMPT_NUDGE): + texts.append(content) + for part in content if isinstance(content, list) else []: + if not isinstance(part, dict): + continue + if str(part.get("type", "")).endswith("tool_use"): + calls.append(part) + if event.get("type") == "assistant" and answer(part): + answers.add(part.get("id")) + elif part.get("type") == "tool_result" and part.get("is_error"): + refused.add(part.get("tool_use_id")) + + structured = [part for part in calls if answer(part)] + other = [part.get("name") or part.get("type") for part in calls if not answer(part)] + if other: + problems.append(f"tool calls {other}") + accepted = [part for part in structured if part.get("id") not in refused] + if len(accepted) != 1: + problems.append(f"{len(accepted)} accepted StructuredOutput calls, not 1") + elif json.dumps(accepted[0].get("input"), sort_keys=True) != json.dumps( + verdict, sort_keys=True + ): + problems.append("its accepted StructuredOutput answer is not verdict.json") + if len(texts) != 1: + problems.append(f"{len(texts)} user text messages, not the prompt alone") + elif text is None or texts[0] != text: + problems.append("its user message is not prompt.md") + if not all( + isinstance(content, list) + and content + and all( + isinstance(part, dict) + and part.get("type") == "tool_result" + and part.get("tool_use_id") in answers + for part in content + ) + for content in others + ): + problems.append("a user event that is not a StructuredOutput call's result") + return problems + + +def withhold_addresses(value): + """``value`` with every e-mail address in it replaced by WITHHELD_ADDRESS.""" + if not isinstance(value, str): + return value + return EMAIL_ADDRESS.sub(WITHHELD_ADDRESS, value) + + +def verify_judge_provenance( + cases_dir: Path, rejudged: frozenset[str], record_path: Path +) -> None: + """The published judge provenance must describe the staged new verdicts. + + The record (JUDGE_PROVENANCE) lists every case Muse Wonder re-opened, each + once, with its verdict's and prompt's sha256, its group and whether its + judge ran isolated, and its sidecar's judge_effort, judge_model_reported, + judged_at_utc and judge_account_declared (JUDGE_SIDECAR_FIELDS), an + e-mail address in any of them withheld as WITHHELD_ADDRESS. The record and + each entry carry exactly their listed keys (JUDGE_PROVENANCE_KEYS, + JUDGE_PROVENANCE_ENTRY_KEYS), its counts are the tally of the entries' + groups, and it names no e-mail address anywhere, for it is public. Each + entry's hashes must be the staged verdict's and prompt's, and its other + fields its sidecar's. Its isolation must be its sidecar's: a sidecar + records judge_isolation only when scripts/run_audit_claude.sh's hardened + runner wrote it, and only an isolated entry's group says "isolated: ". + An isolated verdict's sidecar must record a token login + (judge_auth.method oauth_token), and its transcript, + claude.transcript.jsonl, must pass the runner's transcript checks + (transcript_problems), its prompt and verdict included: the one user + text message must be the staged prompt.md, and the one accepted + StructuredOutput answer the staged verdict.json. + """ + record = json.loads(record_path.read_text()) + entries = record.get("verdicts") if isinstance(record, dict) else None + require( + isinstance(entries, list) and all(isinstance(e, dict) for e in entries), + f"{JUDGE_PROVENANCE_PATH} lists no verdicts", + ) + require( + set(record) == JUDGE_PROVENANCE_KEYS, + f"{JUDGE_PROVENANCE_PATH} has keys {sorted(record)}, not " + f"{sorted(JUDGE_PROVENANCE_KEYS)}", + ) + addresses = account_data(record, "record", keys=False) + require( + not addresses, + f"{JUDGE_PROVENANCE_PATH} is public and names {addresses[:4]}; withhold " + f"each e-mail address as {WITHHELD_ADDRESS!r}", + ) + named = [entry.get("case_id") for entry in entries] + require( + len(named) == len(set(named)) and set(named) == set(rejudged), + f"{JUDGE_PROVENANCE_PATH} does not list each re-opened case once: " + f"{sorted(set(named) ^ set(rejudged))[:4]}", + ) + tally: dict = {} + for entry in entries: + group = entry.get("group") + tally[group] = tally.get(group, 0) + 1 + require( + record["counts"] == tally, + f"{JUDGE_PROVENANCE_PATH} counts {record['counts']!r} are not the tally " + f"of its entries' groups {tally!r}", + ) + wrong = [] + for entry in entries: + case = cases_dir / entry["case_id"] + try: + meta = json.loads((case / "verdict.meta.json").read_text()) + verdict = json.loads((case / "verdict.json").read_text()) + prompt = (case / "prompt.md").read_bytes() + problems = [] + if entry.get("verdict_sha256") != digest(case / "verdict.json"): + problems.append("the record names another verdict") + if entry.get("prompt_sha256") != hashlib.sha256(prompt).hexdigest(): + problems.append("the record names another prompt") + except (OSError, ValueError) as error: + wrong.append(f"{entry['case_id']}: {error}") + continue + if set(entry) != JUDGE_PROVENANCE_ENTRY_KEYS: + problems.append(f"the entry has keys {sorted(entry)}") + meta = meta if isinstance(meta, dict) else {} + for field in JUDGE_SIDECAR_FIELDS: + expected = withhold_addresses(meta.get(field)) + if entry.get(field) != expected: + problems.append( + f"its {field} {entry.get(field)!r} is not the sidecar's " + f"{expected!r}" + ) + isolated = "judge_isolation" in meta + group = entry.get("group") + if not isinstance(group, str) or group.startswith("isolated: ") != isolated: + problems.append(f"its group {group!r} disagrees with its isolation") + if entry.get("isolated") is not isolated: + problems.append( + f"the record says isolated={entry.get('isolated')!r} but the " + f"sidecar {'records' if isolated else 'lacks'} judge_isolation" + ) + elif isolated: + auth = meta.get("judge_auth") + if not isinstance(auth, dict) or auth.get("method") != "oauth_token": + problems.append( + "the sidecar does not record a token login (judge_auth " + f"method oauth_token): {auth!r}" + ) + problems += transcript_problems( + case / "claude.transcript.jsonl", + meta.get("judge_effort"), + prompt, + verdict, + ) + if problems: + wrong.append(f"{entry['case_id']}: {'; '.join(problems)}") + require( + not wrong, + f"{len(wrong)} new verdicts disagree with {JUDGE_PROVENANCE_PATH}: {wrong[:4]}", + ) + + +# The provenance groups of this release's new verdicts, by the account their +# sidecar declares: the pb-judge setup-token login of Max's 2026-09-30 opt-in +# (the GPT-6.1 Sol stage's "setup-token 2"), and subfleet lane claude-18's token +# for those judged after that login's weekly limit (Fleet ops, 2026-10-09). +PROVENANCE_GROUPS = { + "claude setup-token login (token sha256 6a6daf56361b), Max 2026-09-30": ( + "isolated: setup-token 2" + ), + "claude:max@axiom.org (subfleet lane claude-18)": ( + "isolated: subfleet lane claude-18" + ), +} +PROVENANCE_NOTE = ( + "Provenance of the new Opus 5.5 judge verdicts in the Muse Wonder stage: " + "every case Muse Wonder re-opened, which includes the nine cases release " + "20261010 reworded. Every verdict ran through scripts/run_audit_claude.sh " + "from an empty directory outside the repo, with no tools, an allowlisted " + "environment, no user settings and a token login, at effort xhigh; each " + "transcript passes the runner's checks. Most ran on Max's second " + "setup-token login (2026-09-30 opt-in); those judged after that login's " + "weekly limit ran on subfleet lane claude-18's token. Counts are by group. " + "Scores do not depend on judge verdicts." +) + + +def judge_provenance_record(cases_dir: Path, rejudged: frozenset[str]) -> dict: + """The record verify_judge_provenance checks, built from the sidecars.""" + entries = [] + for case_id in sorted(rejudged): + case = cases_dir / case_id + meta = json.loads((case / "verdict.meta.json").read_text()) + declared = meta.get("judge_account_declared") + require( + declared in PROVENANCE_GROUPS, + f"{case_id}: no provenance group for the declared account", + ) + require("judge_isolation" in meta, f"{case_id}: the verdict is not isolated") + entry = { + "case_id": case_id, + "group": PROVENANCE_GROUPS[declared], + "isolated": True, + **{ + field: withhold_addresses(meta.get(field)) + for field in JUDGE_SIDECAR_FIELDS + }, + "verdict_sha256": digest(case / "verdict.json"), + "prompt_sha256": digest(case / "prompt.md"), + } + entries.append(entry) + counts: dict[str, int] = {} + for entry in entries: + counts[entry["group"]] = counts.get(entry["group"], 0) + 1 + return {"note": PROVENANCE_NOTE, "counts": counts, "verdicts": entries} + + +def write_judge_provenance(args) -> None: + """Write JUDGE_PROVENANCE from the stage's sidecars, then check it.""" + cases = args.stage_dir / "audit" / "cases" + rejudged = rejudged_cases(args.stage_dir) + require( + not validate_verdicts(args.stage_dir / "audit", seed=load_seed(args.stage_dir)), + "missing or invalid verdicts; run judge", + ) + # json.dumps(indent=1), as release 20260930's docs/gpt61sol record is. + record = json.loads( + json.dumps(judge_provenance_record(cases, rejudged), sort_keys=True) + ) + JUDGE_PROVENANCE.write_text(json.dumps(record, indent=1, allow_nan=False) + "\n") + verify_judge_provenance(cases, rejudged, JUDGE_PROVENANCE) + print(f"Wrote {JUDGE_PROVENANCE_PATH}: {len(rejudged)} verdicts") + + +def judge(args, bundle) -> None: + """Run one Claude CLI judge at a time and retry missing/hedged cases.""" + from policybench.audit import collect_audit + + audit = args.stage_dir / "audit" + seed = load_seed(args.stage_dir) + for _ in range(3): + validate_verdicts(audit, remove_invalid=True, seed=seed) + subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_claude.sh"), str(audit)], + cwd=ROOT, + env={ + **os.environ, + "AUDIT_MODEL": JUDGE_MODEL, + "AUDIT_PARALLEL": "1", + "AUDIT_PYTHON": sys.executable, + }, + check=True, + ) + pending = validate_verdicts(audit, remove_invalid=True, seed=seed) + out = collect_audit(bundle / "us", audit) + for case_id in out["hedged"].case_id: + set_aside(audit, case_id, "hedged") + if not pending and out["missing"].empty and out["hedged"].empty: + return + raise SystemExit( + "audit missing/hedged after three passes; resume judge after investigation" + ) + + +def triage(args, bundle) -> None: + """Collect, apply recorded decisions, and stop for unresolved flags.""" + from policybench.adjudications import ( + apply_adjudications, + excluded_case_keys, + verify_adjudications_applied, + ) + from policybench.audit import collect_audit + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + audit = args.stage_dir / "audit" + require( + not validate_verdicts(audit, seed=load_seed(args.stage_dir)), + "missing or invalid verdicts; run judge", + ) + out = collect_audit(bundle / "us", audit) + require( + out["missing"].empty and out["hedged"].empty, "missing/hedged audit; run judge" + ) + rows = out["row"] + cases = out["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ) + counts = rows.groupby(KEY).size() + require( + all( + int(r.wrong_model_count) == counts[(r.scenario_id, r.variable)] + for r in cases.itertuples() + ), + "case wrong_model_count disagrees with collected rows", + ) + annotations = bundle / "annotations" + rejudged = rejudged_cases(args.stage_dir) + amendments = load_amendments(args.stage_dir, rejudged) + upgrade = stage_upgrade(args.stage_dir) + if upgrade is not None: + verify_recorded_drops(args.stage_dir, upgrade) + decisions = stage_adjudications( + annotations / ADJUDICATIONS, + rejudged, + amendments, + audit / "cases", + regenerated=frozenset() if upgrade is None else upgrade.regenerated_ruled, + dropped=frozenset() if upgrade is None else upgrade.dropped_cases, + added=upgrade_decisions(upgrade), + triage=triage_items(upgrade), + ) + rows, cases, _ = apply_adjudications(rows, cases, decisions) + amend_annotations(rows, cases, amendments) + # An amendment may not touch the adjudication sentence a case note carries. + verify_adjudications_applied(rows, cases, decisions) + excluded = exclusion_keys(load_reference_exclusions(bundle / "us")) + require( + excluded == excluded_case_keys(decisions), + "adjudication exclusions differ from frozen scoring exclusions", + ) + flags = cases[cases.reference_suspect.astype(bool)] + scored = ~rows[KEY].apply(tuple, axis=1).isin(excluded) + unresolved = rows[ + scored + & ~rows.failure_source.isin( + [ + "llm_error", + "parse_contract_failure", + "budget_exhausted_at_ceiling", + ] + ) + ] + flags.to_csv(args.stage_dir / "reference-flags.csv", index=False) + unresolved.to_csv(args.stage_dir / "unresolved-rows.csv", index=False) + for name, frame in ( + ("us_audit_row_annotations.csv", rows), + ("us_case_notes.csv", cases), + ): + (annotations / name).write_text(annotation_csv_text(frame), encoding="utf-8") + require( + flags.empty and unresolved.empty, + "triage required: inspect reference-flags.csv/unresolved-rows.csv; " + "record evidence in staged us_adjudications.json and rerun triage", + ) + print(f"Triage complete: {len(rows)} rows / {len(cases)} cases") + + +def incumbent_drift(stats: list[dict], previous: dict[str, dict]) -> list[str]: + """Incumbents whose modelStats entry is not byte-identical to the base's. + + Serialized as the payload serializes it, so key order counts as well as + values. A missing incumbent counts as drift. + """ + exported = {row["model"]: row for row in stats} + return sorted( + model + for model, row in previous.items() + if model not in exported + or json.dumps(exported[model], allow_nan=False) + != json.dumps(row, allow_nan=False) + ) + + +def payload_text(payload: dict) -> str: + """The bytes export writes for a payload, and the freeze rebuilds. + + freeze_snapshot reassembles these exact default-json bytes from the + compact country payload. Pretty-printing here breaks its release hash. + """ + return json.dumps(payload, allow_nan=False) + + +def build_payload( + bundle: Path, + live: dict, + *, + partial: bool = False, + early: bool = False, + upgrade: Upgrade | None = None, +) -> dict: + """The payload export writes, built from ``bundle``; the one definition. + + export_full_run's US payload must hold all 46 models. For a release, its + roster must be the base's incumbents plus the addition, Fable 5's usage is + carried from ``live`` (CARRIED_USAGE), and no incumbent may drift; then the + dashboard schema. The freeze rebuilds the staged payload with this. + export_full_run writes data.json, us/data.json and us/analysis/ into + ``bundle``. + + With an engine upgrade installed (``upgrade``) the scope check is two-sided: + (a) with release 20261010's references and exclusion record both put back, + every incumbent's modelStats is release 20261010's byte for byte; (b) + every model is scored on 1,984 outputs less the build's exclusion count. + """ + from policybench.dashboard_schema import validate_dashboard_payload + from policybench.full_run_export import export_full_run + + payload = export_full_run(bundle, countries=["us"], skip_app_data=True) + stats = payload["countries"]["us"]["modelStats"] + require( + len(stats) == BOARD_MODELS, + f"export did not contain all {BOARD_MODELS} models", + ) + previous = {m["model"]: m for m in live["countries"]["us"]["modelStats"]} + if not partial: + require( + len(previous) == BASE_MODELS and not set(previous) & set(MODELS.values()), + f"base must hold the {BASE_MODELS} incumbents only", + ) + require( + {s["model"] for s in stats} == set(previous) | set(MODELS.values()), + "export roster is not the incumbents plus the addition", + ) + for model, keys in CARRIED_USAGE.items(): + row = next(s for s in stats if s["model"] == model) + for key in keys: + row[key] = previous[model][key] + scored = {row["model"]: row["n"] for row in stats} + expected = RELEASE_SCORED if upgrade is None else upgrade.scored_outputs + require( + set(scored.values()) == {expected}, + f"every model must be scored on {expected} outputs: {scored}", + ) + # The scope check: with release 20261010's exclusion record put back, + # every incumbent's modelStats is release 20261010's byte for byte, so + # every incumbent change comes from the ten ruled records. With an + # engine upgrade, its references are put back too, so every change + # comes from the build's references and exclusion record. + import tempfile + + from release_20261006 import export_payload + + with tempfile.TemporaryDirectory() as scratch: + record = Path(scratch) / "reference_exclusions.json" + record.write_text(exclusions_text(base_exclusion_record())) + if upgrade is None: + scoped = export_payload(bundle, live, record) + else: + scoped = export_on_base_references(bundle, live, record) + scoped_stats = scoped["countries"]["us"]["modelStats"] + require(len(scoped_stats) == BOARD_MODELS, "scope export lost a model") + drift = incumbent_drift(scoped_stats, previous) + put_back = ( + "exclusion record" if upgrade is None else "references and exclusion record" + ) + require( + not drift, + f"with release 20261010's {put_back}, incumbent modelStats drift: {drift}", + ) + errors = validate_dashboard_payload(payload, require_failure_annotations=not early) + require(not errors, f"payload validation failed: {errors[:8]}") + if partial: + payload["stage2Status"] = ( + "PARTIAL — incomplete household cohorts, not a release" + ) + return payload + + +def export(args, bundle, live) -> dict: + """Export into scratch, preserve incumbent statistics, and gate release.""" + upgrade = None if args.partial else stage_upgrade(args.stage_dir) + if not args.partial and upgrade is None: + # After the freeze the pointer and committed record are this release's. + release = release_exclusions_sha256() + frozen = live_pointer()["tag"] == RELEASE_TAG + verify_reference_pins( + SNAPSHOT, "committed reference", release if frozen else None + ) + verify_reference_pins(bundle / "us", "staged reference", release) + elif upgrade is not None: + # The staged references are the installed build's (BASE_REFERENCE_SHA256 + # stays release 20261010's); the committed ones are release 20261010's + # until the freeze and the build's after it. + pins = upgraded_reference_pins(upgrade) + frozen = live_pointer()["tag"] == RELEASE_TAG + verify_reference_pins( + SNAPSHOT, "committed reference", pins=pins if frozen else None + ) + verify_reference_pins(bundle / "us", "staged reference", pins=pins) + if not args.early: + verify_new_model_inputs(args.stage_dir) + verify_judge_provenance( + args.stage_dir / "audit" / "cases", + rejudged_cases(args.stage_dir), + JUDGE_PROVENANCE, + ) + payload = build_payload( + bundle, live, partial=args.partial, early=args.early, upgrade=upgrade + ) + stats = payload["countries"]["us"]["modelStats"] + path = args.stage_dir / ( + f"PARTIAL-data-board{BOARD_MODELS}.json" + if args.partial + else f"data-board{BOARD_MODELS}.json" + ) + path.write_text(payload_text(payload)) + # Keep the freeze input identical to the gated payload, including Fable usage. + shutil.copyfile(path, bundle / "data.json") + ranked = sorted(stats, key=lambda s: (-s["exact"], -s["score"], s["model"])) + for index, row in enumerate(ranked, 1): + if row["model"] in MODELS.values(): + print( + f"{'PARTIAL common cohort' if args.partial else 'STAGED'} " + f"{row['model']}: exact={row['exact']:.6f}% " + f"rank={index}/{BOARD_MODELS} n={row['n']}" + ) + if not args.early: + pinned = [p for p in (bundle / "annotations").iterdir() if p.is_file()] + pinned += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + pinned += [p for p in (args.stage_dir / "inputs").rglob("*") if p.is_file()] + # An installed engine upgrade's build, verbatim (absent otherwise). + pinned += [ + p for p in sorted((args.stage_dir / BUILD_COPY).glob("*")) if p.is_file() + ] + # The cases Muse Wonder re-opened, the listed wording amendments and the + # seed binding decide what the adjudication and verdict gates allow; + # stage.json and model-provenance.json hold the prepare-time hashes. + pinned += [ + path + for path in ( + args.stage_dir / PROMPT_CHANGES, + args.stage_dir / AMENDMENTS, + args.stage_dir / "stage.json", + args.stage_dir / "model-provenance.json", + ) + if path.is_file() + ] + pinned += [ + p + for p in (args.stage_dir / "audit").rglob("*") + if p.is_file() + and p.name + in { + "verdict.json", + "verdict.meta.json", + "prompt.md", + "claude.transcript.jsonl", + "cases.jsonl", + "schema.json", + } + ] + write_json( + args.stage_dir / "release-ready.json", + { + "release_tag": RELEASE_TAG, + "base_tag": BASE_TAG, + "base_sha256": BASE_SHA256, + "payload_sha256": digest(path), + "models": BOARD_MODELS, + "partial": False, + "files": { + str(p.relative_to(args.stage_dir)): digest(p) for p in pinned + }, + # A repository file, so outside "files", which binds stage files. + "judge_provenance": { + "path": JUDGE_PROVENANCE_PATH, + "sha256": digest(JUDGE_PROVENANCE), + }, + }, + ) + return payload + + +# --- An engine upgrade of the references (--step install-references) --------- +# +# Max, 2026-09-28: "we should be using the latest pe for this always!" The +# release moves its references from policyengine-us 2.38.6 to the newest +# release. The builder in reference_audit/2026-10-12-engine-upgrade/, adapted +# from release 20261010's (reference_audit/2026-10-09-engine-upgrade/), reads +# release 20261010's references from git, +# recomputes all 1,984 outputs and writes BUILT_FILES into a directory; the +# narratives script regenerates the reference explanation of each output it +# changed. --step install-references gates that build against release +# 20261010 (load_build) and installs it; every later step holds the stage to +# the installed build (verify_installed_references). The build's revision is +# the sidecar's last, kind engine_upgrade, as #182's was. A stage that never +# runs the step is staged exactly as before, on the 20261010 pins. + +BUILT_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "reference_traces.json", +) +CSV_NAME, META_NAME, EXCLUSIONS_NAME, TRACES_NAME = BUILT_FILES +# The reference files a build replaces in the stage's scoring source and +# bundle; scenarios.csv and its sidecar keep their 20261010 pins. +UPGRADED_FILES = (CSV_NAME, META_NAME, EXCLUSIONS_NAME) +EXPLANATIONS_NAME = "us_case_reference_explanations.csv" +ACTIONS_NAME = "final_actions.json" +# The stage directory that keeps the installed build verbatim: the four built +# files, the regenerated explanations and the build's actions. Later steps +# re-gate it, and export binds it. +BUILD_COPY = "reference-build" +BUILD_COPY_FILES = (*BUILT_FILES, EXPLANATIONS_NAME, ACTIONS_NAME) +# Release 20261010's reference explanations, us_case_reference_explanations.csv +# as committed at BASE_COMMIT. +BASE_EXPLANATIONS_SHA256 = ( + "7ef73f3ff003819eee9dd8bf5ba7b961bf27cfc863455be1f790a36f43498f5f" +) +# The engine release 20261010's references were computed on: its sidecar's last +# revision, the 2026-10-10 engine_upgrade. +BASE_ENGINE = "policyengine-us 2.38.6" +ENGINE_PREFIX = "policyengine-us " +# The sidecar fields a build rewrites; every other field must stay 20261010's. +SIDECAR_REWRITTEN = frozenset( + {"policyengine_bundles", "reference_csv_sha256", "regenerated_at_utc", "revisions"} +) +# A regenerated record's output must come back within this of the record's +# alternative_value, the value its exclusion said the law gives. +REGENERATION_TOLERANCE = 1.0 +# An excluded output keeps the value it was decided on (rule 5); the loader's +# own tolerance for a record's frozen_value (verify_exclusions_against_reference). +FROZEN_TOLERANCE = 1e-3 +# How far an approved move's engine value may sit from the value its action +# approves: the builder's APPROVED_TOL, half a cent. +APPROVED_TOLERANCE = 0.005 + + +@dataclass(frozen=True) +class Baseline: + """What a build is gated against: release 20261010's reference files and + explanations, read from git and pinned; the release's ruled records, as + build_release_exclusions spells them (record_edits applied); and the ruled + outputs the spec says an engine upgrade regenerates (spec_regenerated).""" + + files: dict + explanations: bytes + ruled: dict + regenerated: frozenset + + +@dataclass(frozen=True) +class Upgrade: + """An installed, gated engine upgrade of the references. + + ``changed`` maps each output whose reference the build moved to its new + value. ``regenerated_base`` and ``regenerated_ruled`` are the release + 20261010 records and ruled records the build regenerated (scored again); + ``added`` the records it newly excludes. ``records`` counts the build's + exclusion record, ``exclusions_text`` is its bytes, and ``sha256`` binds + each of BUILD_COPY_FILES. + """ + + engine_version: str + previous_engine_version: str + revision: dict + changed: dict + regenerated_base: frozenset + regenerated_ruled: frozenset + added: frozenset + records: int + exclusions_text: str + sha256: dict + + @property + def regenerated(self) -> frozenset: + return self.regenerated_base | self.regenerated_ruled + + @property + def scored_outputs(self) -> int: + return BASE_OUTPUTS - self.records + + @property + def upgraded_cases(self) -> frozenset[str]: + """The audit cases whose reference value or explanation it changed.""" + return frozenset(output_case(key) for key in self.changed) + + @property + def dropped_cases(self) -> frozenset[str]: + """The cases whose release 20261010 decision it drops (#178).""" + return frozenset(output_case(key) for key in self.regenerated_base) + + +def output_case(key: tuple[str, str]) -> str: + """The audit case id of a US output.""" + return f"us__{key[0]}__{key[1]}" + + +def base_reference_bytes(name: str) -> bytes: + """A release 20261010 reference file, read from BASE_COMMIT and pinned.""" + raw = base_commit_blob(Path("paper/snapshot/20260501/runs") / RUN_NAME / name) + require( + hashlib.sha256(raw).hexdigest() == BASE_REFERENCE_SHA256[name], + f"release 20261010's {name} does not match its pin", + ) + return raw + + +def base_explanations_bytes() -> bytes: + """Release 20261010's reference explanations, read from BASE_COMMIT.""" + raw = base_commit_blob(REFERENCE_EXPLANATIONS) + require( + hashlib.sha256(raw).hexdigest() == BASE_EXPLANATIONS_SHA256, + f"release 20261010's {EXPLANATIONS_NAME} does not match its pin", + ) + return raw + + +def spec_regenerated(spec: dict) -> frozenset[tuple[str, str]]: + """The ruled outputs an installed engine upgrade regenerates. + + docs/musewonder/spec.json's regenerated_by_upgrade.outputs names them; every + other ruled output stays excluded (kept). Each must be a ruled record with + reason code reference_engine_defect: an upgrade can fix the engine, never + an unlisted input or a figure published after the freeze. The field is + read only with an upgrade installed; an absent field names none. + """ + from policybench.reference_exclusions import ENGINE_DEFECT + + section = spec.get("regenerated_by_upgrade") + if section is None: + return frozenset() + outputs = section.get("outputs") if isinstance(section, dict) else None + require( + isinstance(outputs, list) + and all( + isinstance(o, list) and len(o) == 2 and all(isinstance(p, str) for p in o) + for o in outputs + ), + "the spec's regenerated_by_upgrade.outputs is not a list of " + "[scenario_id, variable] pairs", + ) + keys = [tuple(output) for output in outputs] + require( + len(set(keys)) == len(keys), + f"the spec's regenerated_by_upgrade names an output twice: {keys}", + ) + ruled = {spec_key(record): record for record in spec_records(spec)} + unknown = sorted(set(keys) - set(ruled)) + require( + not unknown, + f"the spec's regenerated_by_upgrade names outputs it does not rule on: " + f"{unknown}", + ) + other = sorted(k for k in keys if ruled[k]["reason_code"] != ENGINE_DEFECT) + require( + not other, + "an engine upgrade regenerates only an engine-defect record; the spec's " + f"regenerated_by_upgrade names {other}", + ) + return frozenset(keys) + + +def upgrade_baseline(spec: dict | None = None) -> Baseline: + """The Baseline a build is gated against, from git and the spec.""" + spec = load_spec() if spec is None else spec + files = {name: base_reference_bytes(name) for name in UPGRADED_FILES} + release = build_release_exclusions(json.loads(files[EXCLUSIONS_NAME]), spec) + ruled_keys = {spec_key(record) for record in spec_records(spec)} + ruled = { + spec_key(record): record + for record in release["exclusions"] + if spec_key(record) in ruled_keys + } + return Baseline(files, base_explanations_bytes(), ruled, spec_regenerated(spec)) + + +def regeneration_lands(variable: str, value: float, alternative: float) -> bool: + """Whether a regenerated output lands on its audited corrected value: + within REGENERATION_TOLERANCE ($1, the exact-match tolerance) for an + amount, equal for a 0/1 flag (policybench.paper_results' + moves_beyond_tolerance, which the builder applies too).""" + from policybench.paper_results import moves_beyond_tolerance + + return abs(value - alternative) <= REGENERATION_TOLERANCE and not ( + moves_beyond_tolerance(variable, alternative, value) + ) + + +def engine_older(older: str, newer: str) -> bool: + """Whether one 'policyengine-us X.Y.Z' release precedes another.""" + + def parts(engine) -> tuple[int, ...] | None: + if not isinstance(engine, str) or not engine.startswith(ENGINE_PREFIX): + return None + try: + return tuple(int(p) for p in engine.removeprefix(ENGINE_PREFIX).split(".")) + except ValueError: + return None + + a, b = parts(older), parts(newer) + return a is not None and b is not None and a < b + + +# The audited root-cause fix modules a "fix_modules" regeneration target +# names, as committed at BASE_COMMIT. +AUDIT_FIXES = Path("reference_audit/2026-09-22/fixes") +TARGET_KEYS = { + "record": {"kind", "value", "engine"}, + "fix_modules": { + "kind", + "value", + "engine", + "engine_value", + "modules", + "dependencies", + "evidence", + "evidence_sha256", + "value_with_modules", + }, +} + + +def committed_dependencies(names: list[str]) -> list[str]: + """The siblings the named fix modules load, transitively, as committed at + BASE_COMMIT, in the builder's order: the builder's module_dependencies + (policybench.fix_module_closure), re-derived here from the committed + bytes. Raises ClosureError for a module that loads local files any way + but Path(__file__).with_name("").""" + from policybench.fix_module_closure import dependency_closure + + found: list[str] = [] + for name in names: + read = lambda file: base_commit_blob(AUDIT_FIXES / file) # noqa: E731 + for dep in dependency_closure(name, read): + if dep not in names and dep not in found: + found.append(dep) + return found + + +def regeneration_action_problems( + key: tuple[str, str], record: dict, entry, action +) -> list[str]: + """Where a revision's regenerated_exclusions entry departs from the one + reviewed action that asks for it. The builder copies the action's + tolerance, upstream and basis, checks its alternative_value against the + record, and expands its target: a record target to the record's value, a + fix_modules target to the evidence's for the action's modules and + evidence file. So the gates hold the build to what was reviewed, not to + what the revision says was asked.""" + if not isinstance(action, dict): + return [f"{key}: the build's actions do not regenerate it exactly once"] + if not isinstance(entry, dict): + return [f"{key}: the build's revision does not list it exactly once"] + problems = [] + tolerance = action.get("tolerance") + if not _number(tolerance) or not 0 <= tolerance <= REGENERATION_TOLERANCE: + return [f"{key}: the action's tolerance {tolerance!r} is not in [0, $1]"] + if not (_number(entry.get("tolerance")) and entry["tolerance"] == tolerance): + problems.append( + f"{key}: the revision's tolerance {entry.get('tolerance')!r} is not the " + f"action's {tolerance!r}" + ) + if not _same( + action.get("alternative_value"), record.get("alternative_value"), 1e-6 + ): + problems.append(f"{key}: the action's alternative_value is not the record's") + for field in ("upstream", "basis"): + if entry.get(field) != action.get(field): + problems.append(f"{key}: the revision's {field} is not the action's") + asked = action.get("target", {"kind": "record"}) + built = entry.get("target") + if ( + not isinstance(asked, dict) + or not isinstance(built, dict) + or (asked.get("kind") != built.get("kind")) + ): + problems.append(f"{key}: the revision's target is not the kind the action asks") + elif asked.get("kind") == "fix_modules": + ref = asked.get("evidence") + modules = built.get("modules") + names = ( + [m.get("module") for m in modules if isinstance(m, dict)] + if isinstance(modules, list) + else None + ) + if ( + names != asked.get("modules") + or not isinstance(ref, dict) + or ref.get("path") != built.get("evidence") + or ref.get("sha256") != built.get("evidence_sha256") + ): + problems.append( + f"{key}: the revision's fix modules or evidence are not the action's" + ) + return problems + + +def regeneration_target_problems( + key: tuple[str, str], + record: dict, + target, + value: float, + engine: str, + tolerance: float = REGENERATION_TOLERANCE, +) -> list[str]: + """Why a regenerated output's audited target does not hold, re-derived + from what the build cites (the builder's regeneration_target). + + A "record" target is the record's alternative_value on the record's + engine. A "fix_modules" target is the corrected value in a committed + evidence file, at its sha256, computed on an older engine than the build's + with fix modules committed at BASE_COMMIT; there the modules moved the + output beyond the exact-match tolerance (the defect was present), and on + the build's engine they moved it by no more than REGENERATION_TOLERANCE + (nothing is left to fix). Either way the build's value lands on it. + """ + variable = key[1] + if not isinstance(target, dict) or target.get("kind") not in TARGET_KEYS: + return [f"{key}: the regeneration names no audited target"] + kind = target["kind"] + if set(target) != TARGET_KEYS[kind]: + return [f"{key}: the {kind} target's fields are {sorted(target)}"] + if not _number(target["value"]): + return [f"{key}: the target's value is not a number"] + problems = [] + if not _number(tolerance) or not 0 <= tolerance <= REGENERATION_TOLERANCE: + return [f"{key}: the regeneration's tolerance {tolerance!r} is not in [0, $1]"] + if kind == "record": + if not _same(target["value"], float(record["alternative_value"])) or ( + target["engine"] != record["engine_version"] + ): + problems.append(f"{key}: the record target is not the record's") + else: + problems.extend( + _fix_modules_target_problems(key, target, value, engine, tolerance) + ) + if not regeneration_lands(variable, value, float(target["value"])) or ( + abs(value - float(target["value"])) > tolerance + ): + problems.append( + f"{key}: regenerated at {value}, not within ${tolerance:g} of its " + f"audited target {target['value']} ({kind})" + ) + return problems + + +def _fix_modules_target_problems( + key: tuple[str, str], + target: dict, + value: float, + engine: str, + tolerance: float = REGENERATION_TOLERANCE, +) -> list[str]: + from policybench.paper_results import moves_beyond_tolerance + + variable = key[1] + path = target["evidence"] + if ( + not isinstance(path, str) + or Path(path).is_absolute() + or ".." in Path(path).parts + ): + return [f"{key}: the evidence path {path!r} is not in the checkout"] + file = ROOT / path + if not file.is_file() or digest(file) != target["evidence_sha256"]: + return [f"{key}: {path} is missing or not the evidence its sha256 pins"] + try: + doc = json.loads(file.read_text()) + except ValueError: + return [f"{key}: {path} is not JSON"] + problems = [] + if not isinstance(doc, dict) or doc.get("kind") != "regeneration_evidence": + return [f"{key}: {path} is not regeneration evidence"] + if doc.get("engine") != target["engine"] or not engine_older( + target["engine"], engine + ): + problems.append( + f"{key}: the evidence's engine {doc.get('engine')!r} is not the target's, " + f"or is not older than the build's {engine}" + ) + modules = target["modules"] + if not ( + isinstance(modules, list) + and modules + and all(isinstance(m, dict) and set(m) == {"module", "sha256"} for m in modules) + ): + return [*problems, f"{key}: the target lists no fix modules"] + names = [m["module"] for m in modules] + from policybench.fix_module_closure import ClosureError + + try: + closure = committed_dependencies(names) + except ClosureError as error: + return [*problems, f"{key}: {error}"] + expected = [] + for dep in closure: + expected.append( + { + "module": dep, + "sha256": hashlib.sha256( + base_commit_blob(AUDIT_FIXES / dep) + ).hexdigest(), + } + ) + if target["dependencies"] != expected or any( + doc["modules"].get(item["module"]) != item["sha256"] for item in expected + ): + problems.append( + f"{key}: the target and evidence do not pin the siblings its modules load " + f"as committed at {BASE_COMMIT[:12]}: {[i['module'] for i in expected]}" + ) + for entry in modules: + name = entry["module"] + committed = hashlib.sha256(base_commit_blob(AUDIT_FIXES / name)).hexdigest() + if entry["sha256"] != committed or doc["modules"].get(name) != committed: + problems.append( + f"{key}: {name} is not {AUDIT_FIXES / name} as committed at " + f"{BASE_COMMIT[:12]}" + ) + items = [ + item + for item in doc.get("items", []) + if isinstance(item, dict) + and spec_key(item) == key + and item.get("modules") == names + ] + if len(items) != 1: + return [*problems, f"{key}: {path} has {len(items)} items for {names}"] + item = items[0] + if not ( + _same(item.get("engine_value"), target["engine_value"]) + and _same(item.get("corrected_value"), target["value"]) + ): + problems.append(f"{key}: the target's values are not the evidence's") + elif not moves_beyond_tolerance( + variable, float(item["engine_value"]), float(item["corrected_value"]) + ): + problems.append( + f"{key}: on {doc['engine']} the modules do not move it, so the evidence " + "shows no defect" + ) + # On the build's engine the modules move it by no more than the + # regeneration's tolerance, as the builder requires. + after = target["value_with_modules"] + if ( + not _number(after) + or not regeneration_lands(variable, float(after), value) + or abs(float(after) - value) > tolerance + ): + problems.append( + f"{key}: on {engine} the audited fix still moves it: {value} -> {after}" + ) + return problems + + +def _number(value) -> bool: + import math + + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def _same(a, b, tolerance: float = 1e-9) -> bool: + return _number(a) and _number(b) and abs(float(a) - float(b)) <= tolerance + + +def reference_values(blob: bytes) -> dict[tuple[str, str], float]: + """Each output's reference value in a reference CSV.""" + import io + + import pandas as pd + + frame = pd.read_csv(io.BytesIO(blob)) + return { + (str(s), str(v)): float(x) + for s, v, x in zip(frame.scenario_id, frame.variable, frame.value) + } + + +def reference_value_changes(base: bytes, new: bytes) -> dict[tuple[str, str], float]: + """The outputs whose reference value differs between two reference CSVs. + + The CSVs must hold the same outputs in the same order and the same columns, + and every column but ``value`` (impact_weight among them) must be equal on + every row, as PR #182's driver required (reference_revision). Any change + to a value, however small, counts. + """ + import io + + import pandas as pd + + before = pd.read_csv(io.BytesIO(base)) + after = pd.read_csv(io.BytesIO(new)) + require( + list(after.columns) == list(before.columns), + f"reference columns changed: {list(before.columns)} -> {list(after.columns)}", + ) + require( + "value" in before.columns and set(KEY) <= set(before.columns), + f"the reference CSV lacks {KEY} or value", + ) + keys = [tuple(map(str, row)) for row in before[KEY].itertuples(index=False)] + require( + len(after) == len(before) + and [tuple(map(str, row)) for row in after[KEY].itertuples(index=False)] + == keys, + "the reference CSV's outputs or their order changed", + ) + for column in before.columns: + if column in (*KEY, "value"): + continue + same = (before[column] == after[column]) | ( + before[column].isna() & after[column].isna() + ) + moved = [keys[i] for i in range(len(keys)) if not bool(same.iloc[i])] + require(not moved, f"reference {column} changed: {moved[:5]}") + values = after["value"].astype(float) + require(not values.isna().any(), "a reference value is missing") + base_values = before["value"].astype(float) + return { + keys[i]: float(values.iloc[i]) + for i in range(len(keys)) + if float(values.iloc[i]) != float(base_values.iloc[i]) + } + + +def explanation_rows(blob: bytes, label: str) -> tuple[list[str], list[tuple]]: + """A reference explanations CSV's header and its (output, row) pairs, as + text exactly as written.""" + import csv + import io + + try: + rows = list(csv.reader(io.StringIO(blob.decode("utf-8"), newline=""))) + except (UnicodeDecodeError, csv.Error) as error: + raise SystemExit(f"{label} {EXPLANATIONS_NAME} is not a CSV: {error}") + require( + bool(rows) and {"scenario_id", "variable"} <= set(rows[0]), + f"{label} {EXPLANATIONS_NAME} has no scenario_id and variable columns", + ) + header = rows[0] + s, v = header.index("scenario_id"), header.index("variable") + require( + all(len(row) == len(header) for row in rows[1:]), + f"{label} {EXPLANATIONS_NAME} has a row of the wrong width", + ) + return header, [((row[s], row[v]), row) for row in rows[1:]] + + +def explanation_changes( + base: bytes, new: bytes +) -> tuple[frozenset[tuple[str, str]], dict[tuple[str, str], dict]]: + """The outputs whose explanation row differs from release 20261010's, and + every new row by output. + + The new file must have the base's columns and the base's outputs in the + base's order; a row is compared field by field as text, so every row the + narratives did not rewrite must be the base's, byte for byte. + """ + header, before = explanation_rows(base, "release 20261010's") + new_header, after = explanation_rows(new, "the build's") + require( + new_header == header, + f"the reference explanations' columns changed: {header} -> {new_header}", + ) + require( + [key for key, _ in after] == [key for key, _ in before], + "the reference explanations' outputs or their order changed", + ) + differ = frozenset( + key for (key, row), (_, old) in zip(after, before, strict=True) if row != old + ) + return differ, {key: dict(zip(header, row)) for key, row in after} + + +def upgrade_revision(base_meta: dict, meta: dict, csv_sha256: str) -> dict: + """The build's engine_upgrade revision, once its sidecar is shown to be + release 20261010's with exactly that revision added. + + Every field but SIDECAR_REWRITTEN keeps 20261010's value, in 20261010's + order; the sidecar pins the built CSV; release 20261010's revisions stay, + and one engine_upgrade revision follows them, from BASE_ENGINE to a newer + policyengine-us that the sidecar's bundle names; the revision lists its + changes and keeps every excluded output untouched (rule 5). + """ + require( + isinstance(meta, dict) and list(meta) == list(base_meta), + "the build's sidecar does not have release 20261010's fields in order", + ) + moved = sorted( + key + for key in base_meta + if key not in SIDECAR_REWRITTEN + and json.dumps(meta[key]) != json.dumps(base_meta[key]) + ) + require( + not moved, + f"the build's sidecar changes {moved}; a build rewrites only " + f"{sorted(SIDECAR_REWRITTEN)}", + ) + require( + meta["reference_csv_sha256"] == csv_sha256, + "the build's sidecar does not pin the built reference CSV", + ) + require( + isinstance(meta["regenerated_at_utc"], str), + "the build's sidecar does not date its regeneration", + ) + revisions, before = meta["revisions"], base_meta["revisions"] + require( + isinstance(revisions, list) + and len(revisions) == len(before) + 1 + and json.dumps(revisions[:-1]) == json.dumps(before), + "the build's sidecar does not keep release 20261010's revisions and add one", + ) + revision = revisions[-1] + require( + isinstance(revision, dict) and revision.get("kind") == "engine_upgrade", + "the build's added revision is not an engine_upgrade", + ) + previous = before[-1].get("engine_version") + engine = revision.get("engine_version") + require( + isinstance(engine, str) + and engine.startswith(ENGINE_PREFIX) + and engine != previous, + f"the build's revision names no newer engine than {previous}: {engine!r}", + ) + require( + revision.get("previous_engine_version") == previous, + f"the build's revision does not upgrade from {previous}, release " + f"20261010's engine: {revision.get('previous_engine_version')!r}", + ) + bundles = meta["policyengine_bundles"] + require( + isinstance(bundles, dict) + and set(bundles) == set(base_meta["policyengine_bundles"]) + and isinstance(bundles.get("us"), dict) + and bundles["us"].get("model_version") == engine.removeprefix(ENGINE_PREFIX), + f"the build's sidecar bundle does not name {engine}", + ) + require( + revision.get("excluded_outputs_untouched") is True, + "the build's revision does not keep every excluded output untouched", + ) + changed = revision.get("changed") + require( + isinstance(changed, list) + and all( + isinstance(item, dict) + and isinstance(item.get("scenario_id"), str) + and isinstance(item.get("variable"), str) + and _number(item.get("previous")) + and _number(item.get("regenerated")) + for item in changed + ), + "the build's revision does not list each change's output, previous and " + "regenerated value", + ) + rechecked = revision.get("excluded_outputs_rechecked", []) + require( + isinstance(rechecked, list) + and all( + isinstance(item, dict) + and isinstance(item.get("scenario_id"), str) + and isinstance(item.get("variable"), str) + and _number(item.get("kept_value")) + for item in rechecked + ), + "the build's revision lists a malformed excluded_outputs_rechecked entry", + ) + return revision + + +def declared_exclusions(plan: dict) -> dict[tuple[str, str], dict]: + """The records a build's actions newly exclude: new_exclusions, each the + record itself, and audit_exclusions, each carrying its record (#182).""" + require( + all( + isinstance(plan.get(name, []), list) + for name in ("approved", "new_exclusions", "audit_exclusions") + ) + and isinstance(plan.get("excluded_rechecked", []), list), + "the build's actions are not lists of actions", + ) + declared: dict[tuple[str, str], dict] = {} + items = [ + *((item, item) for item in plan.get("new_exclusions", [])), + *((item, item.get("exclusion")) for item in plan.get("audit_exclusions", [])), + ] + for item, record in items: + require( + isinstance(item, dict) and isinstance(record, dict), + "the build's actions list a malformed new exclusion", + ) + key = spec_key(item) + require( + spec_key(record) == key and key not in declared, + f"the build's actions list {key} twice or under another output", + ) + declared[key] = record + return declared + + +def upgrade_exclusion_records( + baseline: Baseline, + text: str, + values: dict[tuple[str, str], float], + changed: dict[tuple[str, str], float], + plan: dict, + engine: str, + revision: dict | None = None, +) -> tuple[list[dict], frozenset, frozenset, frozenset]: + """The build's exclusion record, gated: the release's record. + + It must be release 20261010's records, minus the ones the build + regenerated, plus the spec's kept ruled records, plus the records the + build's actions newly exclude, and nothing else. So the regenerated ruled + records are exactly the spec's regenerated_by_upgrade; each regenerated + record is an engine defect whose output the built CSV now scores within + REGENERATION_TOLERANCE of its audited target, as the revision's + regenerated_exclusions entry states it and regeneration_target_problems + re-derives it (the record's alternative_value, or the corrected value in a + committed fix-module evidence file); a kept record keeps its + bytes and its output its value (rule 5: unlisted in ``changed``); a new + record is its action's, computed on the build's engine and frozen at the + built value. Release 20261010's records keep their order and its last + (the audit's scenario_023 record) stays last. The record is in the + builder's form and passes the loader. Returns the records and the + regenerated base, regenerated ruled and added outputs. + """ + import tempfile + + from policybench.reference_exclusions import ( + ENGINE_DEFECT, + ReferenceExclusionError, + load_reference_exclusions, + ) + + try: + doc = json.loads(text) + except ValueError as error: + raise SystemExit(f"the build's exclusion record is not JSON: {error}") + require( + isinstance(doc, dict) and text == exclusions_text(doc), + "the build's exclusion record is not in the builder's form (duplicate keys " + "or other bytes the gates cannot see); write it with json.dumps(indent=2)", + ) + base_doc = json.loads(baseline.files[EXCLUSIONS_NAME]) + require( + list(doc) == list(base_doc) + and all( + doc[key] == base_doc[key] + for key in base_doc + if key not in ("exclusions", "derivation") + ) + and isinstance(doc["derivation"], str) + and doc["derivation"].strip(), + "the build's exclusion record changes release 20261010's fields outside " + "its records and derivation", + ) + with tempfile.TemporaryDirectory() as scratch: + (Path(scratch) / EXCLUSIONS_NAME).write_text(text) + try: + load_reference_exclusions(Path(scratch)) + except ReferenceExclusionError as error: + raise SystemExit(f"the build's exclusion record fails the loader: {error}") + built = {spec_key(record): record for record in doc["exclusions"]} + base = {spec_key(record): record for record in base_doc["exclusions"]} + known = set(base) | set(baseline.ruled) + regenerated = known - set(built) + added = set(built) - known + regenerated_ruled = frozenset(regenerated & set(baseline.ruled)) + regenerated_base = frozenset(regenerated & set(base)) + require( + regenerated_ruled == baseline.regenerated, + "the build regenerates ruled records " + f"{sorted(regenerated_ruled)}, but the spec's regenerated_by_upgrade " + f"names {sorted(baseline.regenerated)}", + ) + declared = declared_exclusions(plan) + require( + added == set(declared), + "the build's new exclusions are not its actions': unlisted " + f"{sorted(added - set(declared))[:8]}, listed but absent " + f"{sorted(set(declared) - added)[:8]}", + ) + problems = [] + + def dumps(record: dict) -> str: + return json.dumps(record, ensure_ascii=False) + + for key in sorted(set(built) & set(base)): + if dumps(built[key]) != dumps(base[key]): + problems.append(f"{key}: changes release 20261010's record") + for key in sorted(set(built) & set(baseline.ruled)): + if dumps(built[key]) != dumps(baseline.ruled[key]): + problems.append(f"{key}: changes the release's ruled record") + for key in sorted(set(built) & known): + frozen = float(built[key]["frozen_value"]) + if key in changed or abs(values[key] - frozen) > FROZEN_TOLERANCE: + problems.append( + f"{key}: an excluded output keeps the value it was decided on " + f"({frozen}), but the build gives {values[key]}" + ) + for key in sorted(added): + record = built[key] + if dumps(record) != dumps(declared[key]): + problems.append(f"{key}: the new record is not its action's") + if record.get("engine_version") != engine: + problems.append(f"{key}: the new record is not computed on {engine}") + if abs(values[key] - float(record["frozen_value"])) > FROZEN_TOLERANCE: + problems.append(f"{key}: the new record's frozen_value is not the build's") + for key in sorted(regenerated): + record = base.get(key) or baseline.ruled[key] + if record["reason_code"] != ENGINE_DEFECT: + problems.append( + f"{key}: an engine upgrade regenerates only an engine defect, not " + f"a {record['reason_code']} record" + ) + entries = [ + item + for item in (revision or {}).get("regenerated_exclusions", []) + if isinstance(item, dict) and spec_key(item) == key + ] + actions = [ + item + for item in plan.get("regenerated_exclusions", []) + if isinstance(item, dict) and spec_key(item) == key + ] + entry = entries[0] if len(entries) == 1 else None + action = actions[0] if len(actions) == 1 else None + bound = regeneration_action_problems(key, record, entry, action) + if bound: + problems.extend(bound) + continue + # The reviewed action's tolerance, which the revision restates. + problems.extend( + regeneration_target_problems( + key, record, entry["target"], values[key], engine, action["tolerance"] + ) + ) + order = [key for key in map(spec_key, doc["exclusions"]) if key in base] + if order != [key for key in map(spec_key, base_doc["exclusions"]) if key in built]: + problems.append("release 20261010's records changed their order") + if base_doc["exclusions"]: + tail = spec_key(base_doc["exclusions"][-1]) + if tail in built and spec_key(doc["exclusions"][-1]) != tail: + problems.append(f"release 20261010's last record {tail} is not last") + require(not problems, f"the build's exclusion record: {problems[:8]}") + return doc["exclusions"], regenerated_base, regenerated_ruled, frozenset(added) + + +def load_build( + built: Path, explanations: Path, actions: Path, baseline: Baseline | None = None +) -> Upgrade: + """Gate a reference build against release 20261010 and describe it. + + The build's base must be the pinned release 20261010: its sidecar is + 20261010's with one engine_upgrade revision added (upgrade_revision). Its + reference CSV differs from 20261010's exactly in that revision's changed + list, at the listed values (reference_value_changes); its exclusion record + is the release's (upgrade_exclusion_records); its traces cover every + changed output; its explanations differ from 20261010's only on changed + outputs, each row stating its new value (explanation_changes); and its + actions name its engine, approve only listed moves at their values, and + recheck exactly the excluded outputs the revision rechecks, reasons + verbatim. + """ + for path in [*(built / name for name in BUILT_FILES), explanations, actions]: + require(path.is_file(), f"the reference build lacks {path}") + baseline = upgrade_baseline() if baseline is None else baseline + blobs = {name: (built / name).read_bytes() for name in BUILT_FILES} + blobs[EXPLANATIONS_NAME] = explanations.read_bytes() + blobs[ACTIONS_NAME] = actions.read_bytes() + try: + meta = json.loads(blobs[META_NAME]) + traces = json.loads(blobs[TRACES_NAME]) + plan = json.loads(blobs[ACTIONS_NAME]) + text = blobs[EXCLUSIONS_NAME].decode("utf-8") + except ValueError as error: + raise SystemExit(f"the reference build's records do not parse: {error}") + require(isinstance(plan, dict), "the build's actions are not an object") + revision = upgrade_revision( + json.loads(baseline.files[META_NAME]), + meta, + hashlib.sha256(blobs[CSV_NAME]).hexdigest(), + ) + engine = revision["engine_version"] + base_values = reference_values(baseline.files[CSV_NAME]) + values = reference_values(blobs[CSV_NAME]) + changed = reference_value_changes(baseline.files[CSV_NAME], blobs[CSV_NAME]) + listed: dict[tuple[str, str], dict] = {} + for item in revision["changed"]: + key = (item["scenario_id"], item["variable"]) + require(key not in listed, f"the build's revision lists {key} twice") + listed[key] = item + require( + set(changed) == set(listed), + "the reference CSV differs from release 20261010's outside the build's " + f"changed list: unlisted moves {sorted(set(changed) - set(listed))[:8]}, " + f"listed changes that did not happen {sorted(set(listed) - set(changed))[:8]}", + ) + misstated = sorted( + key + for key, item in listed.items() + if not _same(item["regenerated"], changed[key]) + or not _same(item["previous"], base_values[key]) + ) + require( + not misstated, + f"the build's changed list misstates the previous or regenerated value of " + f"{misstated[:8]}", + ) + records, regenerated_base, regenerated_ruled, added = upgrade_exclusion_records( + baseline, text, values, changed, plan, engine, revision + ) + expected_traces = {f"{s}|{v}" for s, v in changed} + require( + isinstance(traces, dict) + and set(traces) == expected_traces + and all( + isinstance(item, dict) + and isinstance(item.get("pe_variable"), str) + and isinstance(item.get("trace"), str) + for item in traces.values() + ), + "the build's traces do not cover exactly its changed outputs: " + f"{sorted(set(traces) ^ expected_traces)[:8]}", + ) + rewritten, rows = explanation_changes( + baseline.explanations, blobs[EXPLANATIONS_NAME] + ) + outside = sorted(rewritten - set(changed)) + require( + not outside, + f"the build's explanations rewrite outputs it did not change: {outside[:8]}", + ) + stale = sorted( + key + for key in changed + if ( + "reference_value" in rows[key] + and not _same(_float(rows[key]["reference_value"]), changed[key], 1e-6) + ) + or not rows[key].get("explanation", "x").strip() + ) + require( + not stale, + "the build's explanations do not state the new reference of " + f"{stale[:8]}; run the narratives for every changed output", + ) + require( + plan.get("engine") == engine, + f"the build's actions name {plan.get('engine')!r}, not {engine}", + ) + approved = {} + for item in plan.get("approved", []): + require(isinstance(item, dict), "the build's actions list a malformed move") + approved[spec_key(item)] = item + off = sorted( + key + for key, item in approved.items() + if key not in changed + or not _same(item.get("value"), changed[key], APPROVED_TOLERANCE) + ) + require( + not off, + f"the build's actions approve moves the build did not make: {off[:8]}", + ) + # Every scored reference the build moves beyond the exact-match tolerance + # needs an approval: a regenerated or newly excluded output has its own + # record, and a move within $1 none. + from policybench.paper_results import moves_beyond_tolerance + + unapproved = sorted( + key + for key, new in changed.items() + if key not in regenerated_base | regenerated_ruled | added + and moves_beyond_tolerance(key[1], base_values[key], new) + and key not in approved + ) + require( + not unapproved, + "the build moves scored references beyond the tolerance without an " + f"approval in its actions: {unapproved[:8]}", + ) + rechecked = { + spec_key(item): item for item in revision.get("excluded_outputs_rechecked", []) + } + planned = {spec_key(item): item for item in plan.get("excluded_rechecked", [])} + excluded = {spec_key(record) for record in records} + require( + set(rechecked) == set(planned), + "the build's revision and actions recheck different excluded outputs: " + f"{sorted(set(rechecked) ^ set(planned))[:8]}", + ) + wrong = sorted( + key + for key, item in rechecked.items() + if key not in excluded + or not _same(item["kept_value"], values[key], FROZEN_TOLERANCE) + or ("reason" in planned[key] and planned[key]["reason"] != item.get("reason")) + ) + require( + not wrong, + "the build rechecks outputs it does not keep excluded at their value, or " + f"with another reason: {wrong[:8]}", + ) + claims = builder_claim_problems( + baseline, + revision, + plan, + values, + regenerated_base | regenerated_ruled, + regenerated_ruled, + added, + hashlib.sha256(blobs[ACTIONS_NAME]).hexdigest(), + ) + require( + not claims, + f"the build's account of the upgrade disagrees with its records: {claims[:8]}", + ) + return Upgrade( + engine_version=engine, + previous_engine_version=revision["previous_engine_version"], + revision=revision, + changed=changed, + regenerated_base=regenerated_base, + regenerated_ruled=regenerated_ruled, + added=added, + records=len(records), + exclusions_text=text, + sha256={name: hashlib.sha256(blob).hexdigest() for name, blob in blobs.items()}, + ) + + +def builder_claim_problems( + baseline: Baseline, + revision: dict, + plan: dict, + values: dict[tuple[str, str], float], + regenerated: frozenset, + regenerated_ruled: frozenset, + added: frozenset, + actions_sha256: str, +) -> list[str]: + """Where the builder's own account of the upgrade disagrees with the + records: reference_audit/2026-10-12-engine-upgrade/scripts/ + build_references_upgrade.py lists the regenerated records (each with the + record it removed and its new value), the ruled records kept, and the new + exclusions, in its revision and its actions, and its revision's + provenance names its actions' sha256 and release 20261010's commit.""" + + def outputs(items) -> set: + return {spec_key(item) for item in items if isinstance(item, dict)} + + kept = set(baseline.ruled) - regenerated_ruled + stated = { + "the revision's regenerated_exclusions": ( + revision.get("regenerated_exclusions", []), + regenerated, + ), + "the actions' regenerated_exclusions": ( + plan.get("regenerated_exclusions", []), + regenerated, + ), + "the revision's kept_exclusions_from_release": ( + revision.get("kept_exclusions_from_release", []), + kept, + ), + "the actions' kept_exclusions_from_release": ( + plan.get("kept_exclusions_from_release", []), + kept, + ), + "the revision's new_exclusions": (revision.get("new_exclusions", []), added), + } + problems = [] + for label, (items, want) in stated.items(): + if not isinstance(items, list): + problems.append(f"{label} is not a list") + elif outputs(items) != set(want) or len(items) != len(want): + problems.append(f"{label} names {sorted(outputs(items) ^ set(want))[:4]}") + base = { + spec_key(record): record + for record in json.loads(baseline.files[EXCLUSIONS_NAME])["exclusions"] + } + for item in revision.get("regenerated_exclusions", []): + if not isinstance(item, dict): + continue + key = spec_key(item) + record = base.get(key) or baseline.ruled.get(key) + if record is not None and json.dumps(item.get("record")) != ( + json.dumps(record) + ): + problems.append(f"{key}: the revision's record is not the removed record") + # The builder writes a value that moved by no more than 1e-6 as it was. + if key in values and not _same(item.get("regenerated"), values[key], 1e-6): + problems.append(f"{key}: the revision's regenerated value is not the CSV's") + provenance = revision.get("provenance") + if ( + not isinstance(provenance, dict) + or provenance.get("actions_sha256") != actions_sha256 + or provenance.get("base_commit") != BASE_COMMIT + ): + problems.append( + "the revision's provenance does not name these actions and release " + f"20261010's commit {BASE_COMMIT[:12]}" + ) + if plan.get("previous_engine") != revision["previous_engine_version"]: + problems.append( + f"the actions upgrade from {plan.get('previous_engine')!r}, not " + f"{revision['previous_engine_version']}" + ) + return problems + + +def _float(text: str) -> float | None: + try: + return float(text) + except (TypeError, ValueError): + return None + + +def upgraded_reference_pins(upgrade: Upgrade) -> dict[str, str]: + """Release 20261010's reference pins with the build's three files'.""" + return { + **BASE_REFERENCE_SHA256, + **{name: upgrade.sha256[name] for name in UPGRADED_FILES}, + } + + +def grounding_lookup(path: Path) -> dict[tuple[str, str], str]: + """The audit grounding by output, as prepare_audit takes it.""" + import pandas as pd + + grounding = pd.read_csv(path) + return { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + + +@contextlib.contextmanager +def object_strings(): + """Render prompts as prepare rendered the stage's: resolve_base turns + pandas 3's inferred Arrow strings off before prepare runs.""" + import pandas as pd + + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + with pd.option_context("future.infer_string", False): + yield + else: + yield + + +def render_audit_with( + bundle: Path, sources: dict[str, bytes], lookup: dict +) -> list[tuple[str, str, dict]]: + """Each audit case (id, prompt, manifest row) the bundle gives once the + build's references and explanations are in it, rendered in memory. + + A scratch run directory links every other bundle file and holds the + build's, so nothing in the stage changes. The order is prepare_audit's. + Every prompt is on JUDGE_TEMPLATE_VERSION, the version of every verdict + this release's stage holds; install-references refuses a stage whose + prepare_audit rendering differs. + """ + import tempfile + + from policybench.audit import build_audit_cases, render_case_prompt + + with tempfile.TemporaryDirectory(prefix="install-references-") as scratch: + run = Path(scratch) / bundle.name + for part in ("us", "annotations"): + (run / part).mkdir(parents=True) + if (bundle / part).is_dir(): + for path in sorted((bundle / part).iterdir()): + if path.is_file(): + os.symlink(path.resolve(), run / part / path.name) + replaced = { + **{run / "us" / name: sources[name] for name in UPGRADED_FILES}, + run / "annotations" / EXPLANATIONS_NAME: sources[EXPLANATIONS_NAME], + } + for path, blob in replaced.items(): + path.unlink(missing_ok=True) + path.write_bytes(blob) + with object_strings(): + cases = build_audit_cases(run / "us", grounding_lookup=lookup) + return [ + ( + case.case_id, + render_case_prompt(case, template_version=JUDGE_TEMPLATE_VERSION), + case.to_manifest_row(), + ) + for case in cases + ] + + +def verify_install_target(stage: Path, receipt: dict) -> None: + """The stage's references are release 20261010's (its exclusion record + perhaps the release's, from install-exclusions) or an earlier install's, + in the scoring source and the bundle; its explanations likewise.""" + previous = (receipt.get("references_installed") or {}).get("sha256", {}) + for name in UPGRADED_FILES: + allowed = {BASE_REFERENCE_SHA256[name]} + if name in previous: + allowed.add(previous[name]) + if name == EXCLUSIONS_NAME: + allowed.add(release_exclusions_sha256()) + for target in (stage / "scoring", stage / "publish" / RUN_NAME / "us"): + path = target / name + require( + path.is_file() and digest(path) in allowed, + f"the stage's {path.relative_to(stage)} is neither release " + "20261010's nor an installed build's; prepare a new stage", + ) + path = stage / "publish" / RUN_NAME / "annotations" / EXPLANATIONS_NAME + allowed = {BASE_EXPLANATIONS_SHA256, previous.get(EXPLANATIONS_NAME)} + require( + path.is_file() and digest(path) in allowed, + f"the stage's {EXPLANATIONS_NAME} is neither release 20261010's nor an " + "installed build's; prepare a new stage", + ) + + +def references_record(upgrade: Upgrade, previous: dict | None) -> dict: + """What stage.json records of an installed build (references_installed). + + Installing the same build again records the same thing; installing another + keeps the one it replaces under ``superseded``. + """ + + def keys(outputs) -> list[list[str]]: + return [list(key) for key in sorted(outputs)] + + record = { + "engine_version": upgrade.engine_version, + "previous_engine_version": upgrade.previous_engine_version, + "actions_sha256": upgrade.sha256[ACTIONS_NAME], + "sha256": dict(upgrade.sha256), + "base_sha256": { + **{name: BASE_REFERENCE_SHA256[name] for name in UPGRADED_FILES}, + EXPLANATIONS_NAME: BASE_EXPLANATIONS_SHA256, + }, + "changed": keys(upgrade.changed), + "regenerated_base": keys(upgrade.regenerated_base), + "regenerated_ruled": keys(upgrade.regenerated_ruled), + "added": keys(upgrade.added), + "records": upgrade.records, + "scored_outputs": upgrade.scored_outputs, + "spec_sha256": digest(ROOT / SPEC_PATH), + } + superseded = list((previous or {}).get("superseded", [])) + if previous and {k: v for k, v in previous.items() if k != "superseded"} != record: + superseded.append({k: v for k, v in previous.items() if k != "superseded"}) + record["superseded"] = superseded + return record + + +def bind_build_exclusions(stage: Path, receipt: dict, upgrade: Upgrade) -> None: + """Write the build's exclusion record into the stage and bind it as the + release's (exclusions_installed); the receipt is written by the caller.""" + blob = upgrade.exclusions_text.encode("utf-8") + for target in (stage / "scoring", stage / "publish" / RUN_NAME / "us"): + (target / EXCLUSIONS_NAME).write_bytes(blob) + receipt["files"][f"publish/{RUN_NAME}/us/{EXCLUSIONS_NAME}"] = upgrade.sha256[ + EXCLUSIONS_NAME + ] + receipt["exclusions_installed"] = { + "base_sha256": BASE_REFERENCE_SHA256[EXCLUSIONS_NAME], + "sha256": upgrade.sha256[EXCLUSIONS_NAME], + "spec_sha256": digest(ROOT / SPEC_PATH), + "records": upgrade.records, + "source": f"{BUILD_COPY}/{EXCLUSIONS_NAME}", + "engine_version": upgrade.engine_version, + } + + +def install_references(args) -> None: + """Install a gated engine upgrade of the references into the stage. + + load_build gates the build against release 20261010 before anything is + written, and the audit the new references give is rendered in memory and + held to check_prompt_changes' rule, so a build that would re-open a case + nothing explains is refused first. Then the build goes, verbatim, into + BUILD_COPY; its three reference files into the scoring source and the + bundle, its explanations into the bundle's annotations; stage.json binds + them (files, references_installed with the build's sha256s, engine and + actions, and exclusions_installed for the build's record). Then + policybench.audit.prepare_audit runs again in place on stage/audit with + the pinned grounding: every verdict whose prompt changes (or whose case + leaves the audit) is set aside first, with its judge evidence, in + rejected-verdicts/, and every other verdict keeps its bytes. Last, + prompt-changes.json and pending.json are written anew. Installing the same + build again changes nothing; another build replaces it. + """ + stage = args.stage_dir + require( + digest(args.grounding) == GROUNDING_SHA256, + "grounding differs from the one the 20260929 audit was rendered with", + ) + receipt_path = stage / "stage.json" + receipt = json.loads(receipt_path.read_text()) + require( + "seed" in receipt, + "stage.json does not bind the audit seed; run --step bind-seed first", + ) + seed = receipt["seed"] + verify_seed(seed) + upgrade = load_build(args.built, args.explanations, args.actions) + verify_install_target(stage, receipt) + sources = {name: (args.built / name).read_bytes() for name in BUILT_FILES} + sources[EXPLANATIONS_NAME] = args.explanations.read_bytes() + sources[ACTIONS_NAME] = args.actions.read_bytes() + require( + {name: hashlib.sha256(blob).hexdigest() for name, blob in sources.items()} + == upgrade.sha256, + "the reference build changed while it was gated", + ) + bundle = stage / "publish" / RUN_NAME + audit = stage / "audit" + lookup = grounding_lookup(args.grounding) + rendered = render_audit_with(bundle, sources, lookup) + texts = { + case: text for case, text, row in rendered if not row["parse_failure_only"] + } + seeded = {case: item["prompt_sha256"] for case, item in seed.items()} + upgraded = upgrade.upgraded_cases + changes = classify_prompt_changes( + [row for _, _, row in rendered], + { + case: hashlib.sha256(text.encode()).hexdigest() + for case, text in texts.items() + }, + seeded, + reworded_since_seed(), + upgraded, + ) + reopen, kept = {}, {} + for case in sorted((audit / "cases").iterdir()): + if not case.is_dir() or not ( + (case / "verdict.json").exists() or (case / "verdict.meta.json").exists() + ): + continue + prompt = case / "prompt.md" + if case.name not in texts: + reopen[case.name] = ( + f"install-references: the case leaves the audit on " + f"{upgrade.engine_version}" + ) + elif not prompt.is_file() or prompt.read_text() != texts[case.name]: + reopen[case.name] = ( + "install-references: its prompt re-renders on " + f"{upgrade.engine_version}, whose reference value or explanation " + "it shows" + ) + elif (case / "verdict.json").is_file(): + kept[case.name] = digest(case / "verdict.json") + + # Install the build and bind it before the audit is touched: a stage left + # partway is resumed by installing the same build again. + copy = stage / BUILD_COPY + copy.mkdir(exist_ok=True) + for name, blob in sources.items(): + (copy / name).write_bytes(blob) + for target in (stage / "scoring", bundle / "us"): + for name in UPGRADED_FILES: + (target / name).write_bytes(sources[name]) + (bundle / "annotations").mkdir(exist_ok=True) + (bundle / "annotations" / EXPLANATIONS_NAME).write_bytes(sources[EXPLANATIONS_NAME]) + for name in UPGRADED_FILES: + receipt["files"][f"publish/{RUN_NAME}/us/{name}"] = upgrade.sha256[name] + receipt["files"][f"publish/{RUN_NAME}/annotations/{EXPLANATIONS_NAME}"] = ( + upgrade.sha256[EXPLANATIONS_NAME] + ) + receipt["references_installed"] = references_record( + upgrade, receipt.get("references_installed") + ) + bind_build_exclusions(stage, receipt, upgrade) + pending_receipt = receipt_path.with_name("stage.json.installing") + write_json(pending_receipt, receipt) + os.replace(pending_receipt, receipt_path) + + from policybench.audit import prepare_audit + + for case, reason in reopen.items(): + set_aside(audit, case, reason) + with object_strings(): + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) + manifest = [ + json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() + ] + require( + manifest == [row for _, _, row in rendered] + and all( + (audit / "cases" / case / "prompt.md").read_text() == text + for case, text in texts.items() + ), + "prepare_audit rendered the stage's audit unlike the in-memory rendering", + ) + moved = sorted( + case + for case, sha in kept.items() + if not (audit / "cases" / case / "verdict.json").is_file() + or digest(audit / "cases" / case / "verdict.json") != sha + ) + require(not moved, f"verdicts whose prompts did not change moved: {moved[:8]}") + on_disk = check_prompt_changes(audit, seeded, upgraded) + require( + on_disk == changes, "the written audit's prompt changes are not the rendered" + ) + write_json( + stage / PROMPT_CHANGES, {key: sorted(value) for key, value in changes.items()} + ) + pending = validate_verdicts(audit, remove_invalid=True, seed=seed) + write_json(stage / "pending.json", pending) + verify_prompt_changes(stage, seed) + print( + f"Installed the {upgrade.engine_version} references: {len(upgrade.changed)} " + f"outputs changed, {len(upgrade.regenerated)} exclusions regenerated, " + f"{len(upgrade.added)} added, {upgrade.records} records " + f"({upgrade.scored_outputs} scored). Audit: {len(changes['kept'])} prompts " + f"unchanged, {len(changes['changed'])} changed and {len(changes['added'])} " + f"new; {len(reopen)} verdicts set aside; {len(pending)} cases need Opus 5.5" + ) + + +def verify_installed_references(stage: Path, receipt: dict) -> Upgrade: + """The stage's installed engine upgrade, gated again. + + The build kept in BUILD_COPY must be the bytes stage.json records, must + pass load_build against git and the spec as they are now, must name the + recorded engine, and the stage's scoring source, bundle and explanations + must be its files. + """ + installed = receipt["references_installed"] + copy = stage / BUILD_COPY + pins = installed.get("sha256") + require( + isinstance(pins, dict) and set(pins) == set(BUILD_COPY_FILES), + "stage.json's references_installed does not bind the build's files", + ) + for name, pin in pins.items(): + require( + (copy / name).is_file() and digest(copy / name) == pin, + f"the installed build's {name} changed; run --step install-references " + "again", + ) + upgrade = load_build(copy, copy / EXPLANATIONS_NAME, copy / ACTIONS_NAME) + require( + upgrade.engine_version == installed.get("engine_version") + and upgrade.sha256 == pins, + "the installed build is not the one stage.json records", + ) + bundle = stage / "publish" / RUN_NAME + targets = [stage / "scoring" / name for name in UPGRADED_FILES] + targets += [bundle / "us" / name for name in UPGRADED_FILES] + for path in targets: + require( + path.is_file() and digest(path) == pins[path.name], + f"the stage's {path.relative_to(stage)} is not the installed build's; " + "run --step install-references again", + ) + path = bundle / "annotations" / EXPLANATIONS_NAME + require( + path.is_file() and digest(path) == pins[EXPLANATIONS_NAME], + f"the stage's {EXPLANATIONS_NAME} is not the installed build's", + ) + return upgrade + + +def stage_upgrade(stage: Path) -> Upgrade | None: + """The stage's installed engine upgrade, re-gated, or None without one.""" + path = stage / "stage.json" + if not path.is_file(): + return None + receipt = json.loads(path.read_text()) + if not receipt.get("references_installed"): + return None + return verify_installed_references(stage, receipt) + + +def upgraded_cases(stage: Path) -> frozenset[str]: + """The cases whose reference value or explanation the installed engine + upgrade changed, derived like reworded_since_seed: from the staged + references and explanations against release 20261010's in git. + + They must be exactly the installed revision's changed outputs (each + explanation the narratives rewrote among them). Without an install, none. + """ + receipt = json.loads((stage / "stage.json").read_text()) + if not receipt.get("references_installed"): + return frozenset() + bundle = stage / "publish" / RUN_NAME + meta = json.loads((bundle / "us" / META_NAME).read_text()) + revision = (meta.get("revisions") or [{}])[-1] + listed = { + (item["scenario_id"], item["variable"]) + for item in revision.get("changed", []) + if revision.get("kind") == "engine_upgrade" + } + values = reference_value_changes( + base_reference_bytes(CSV_NAME), (bundle / "us" / CSV_NAME).read_bytes() + ) + texts, _ = explanation_changes( + base_explanations_bytes(), + (bundle / "annotations" / EXPLANATIONS_NAME).read_bytes(), + ) + require( + set(values) == listed and texts <= listed, + "the staged references or explanations differ from release 20261010's " + "outside the installed revision's changed list: " + f"{sorted((set(values) | texts) ^ listed)[:8]}; run --step " + "install-references again", + ) + return frozenset(output_case(key) for key in set(values) | texts) + + +def drop_regenerated_adjudications( + record: dict, upgrade: Upgrade +) -> tuple[dict, list[dict]]: + """``record`` without the decisions of the release 20261010 records the + upgrade regenerated, and what was dropped, as stage.json records it. + + Each regenerated record has exactly one decision, which excludes it; it + is dropped outright, as #178 dropped scenario_045 SNAP's. + """ + dropped, kept = [], [] + for entry in record["adjudications"]: + if case_id(entry) not in upgrade.dropped_cases: + kept.append(entry) + continue + require( + entry.get("excluded_from_scoring") is True, + f"{case_id(entry)}: the decision on a regenerated record does not " + "exclude it", + ) + dropped.append( + { + "case_id": case_id(entry), + "reason": ( + f"Regenerated on {upgrade.engine_version}: the upgrade fixes " + "the engine defect behind this exclusion, so its record leaves " + "the exclusion record and this decision leaves the " + "adjudication record, as release 20260922c dropped " + "scenario_045 SNAP's (#178)." + ), + "entry": entry, + } + ) + named = [item["case_id"] for item in dropped] + require( + len(named) == len(set(named)) and set(named) == upgrade.dropped_cases, + "the staged record does not hold one decision for each regenerated " + f"record: {sorted(set(named) ^ upgrade.dropped_cases)[:8]}", + ) + return {**record, "adjudications": kept}, dropped + + +def verify_recorded_drops(stage: Path, upgrade: Upgrade) -> None: + """stage.json records the drop of exactly the decisions of the records the + upgrade regenerated, each release 20261010's outside its judge fields.""" + from restate_gpt61sol_adjudications import JUDGE_FIELDS + + receipt = json.loads((stage / "stage.json").read_text()) + recorded = receipt.get("adjudications_dropped") + require( + isinstance(recorded, list) and all(isinstance(i, dict) for i in recorded), + "stage.json records no dropped adjudications; run --step adjudicate-exclusions", + ) + named = [item.get("case_id") for item in recorded] + require( + len(named) == len(set(named)) and set(named) == upgrade.dropped_cases, + "stage.json's dropped adjudications are not the regenerated records': " + f"{sorted(set(named) ^ upgrade.dropped_cases)[:8]}", + ) + base = { + case_id(entry): entry for entry in base_adjudication_record()["adjudications"] + } + + def outside_judge(entry: dict) -> dict: + # stage.json is written with sorted keys, so compare values only. + return {k: v for k, v in entry.items() if k not in JUDGE_FIELDS} + + wrong = [ + item["case_id"] + for item in recorded + if not isinstance(item.get("entry"), dict) + or item["case_id"] not in base + or case_id(item["entry"]) != item["case_id"] + or outside_judge(item["entry"]) != outside_judge(base[item["case_id"]]) + ] + require( + not wrong, + f"stage.json records dropped decisions that are not release 20261010's: " + f"{wrong[:8]}", + ) + + +def export_on_base_references(bundle: Path, live: dict, record: Path) -> dict: + """release_20261006.export_payload on a scratch copy of the bundle's export + inputs with release 20261010's five reference files put back (from git) + and ``record`` as the exclusion record: the upgrade's scope export, read + only for its modelStats. + + The dashboard schema is checked without requiring a failure annotation on + every wrong answer: the stage's annotations follow the upgraded + references, so an answer the new reference makes right and the old one + wrong has none here. The release's own payload, on the upgraded + references, is checked with that requirement (build_payload). + """ + import tempfile + + from release_20261006 import EXPORT_INPUTS, export_payload + + with tempfile.TemporaryDirectory(dir=bundle.parent, prefix="base-refs-") as scratch: + copy_root = Path(scratch) / bundle.name + for rel in EXPORT_INPUTS: + (copy_root / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(bundle / rel, copy_root / rel) + for name in REFERENCE_FILES: + (copy_root / "us" / name).write_bytes(base_reference_bytes(name)) + return export_payload( + copy_root, live, record, require_failure_annotations=False + ) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs-root", type=Path) + parser.add_argument("--stage-dir", type=Path) + parser.add_argument( + "--base-predictions", type=Path, default=SNAPSHOT / "predictions.csv.gz" + ) + parser.add_argument("--base-payload", type=Path, default=SNAPSHOT / "data.json.gz") + parser.add_argument("--audit-seed", type=Path) + parser.add_argument("--grounding", type=Path) + parser.add_argument( + "--built", + type=Path, + help="install-references: the reference build's output directory " + f"({', '.join(BUILT_FILES)})", + ) + parser.add_argument( + "--explanations", + type=Path, + help=f"install-references: the build's regenerated {EXPLANATIONS_NAME}", + ) + parser.add_argument( + "--actions", + type=Path, + help=f"install-references: the build's {ACTIONS_NAME}", + ) + parser.add_argument( + "--early", action="store_true", help="fold and export without paid judging" + ) + parser.add_argument( + "--partial", + action="store_true", + help="scratch-only cohort rehearsal; requires --early", + ) + parser.add_argument( + "--step", + choices=( + "pin-inputs", + "prepare", + "bind-seed", + "install-references", + "install-exclusions", + "judge", + "adjudicate-exclusions", + "provenance", + "triage", + "export", + ), + default="prepare", + help="pin-inputs writes the finished run's file hashes to " + f"{INPUT_PINS_PATH} (no stage); bind-seed binds the audit seed in the " + "stage.json of a stage prepared before prepare bound it; " + "install-references installs an engine upgrade of the references " + "(--built, --explanations, --actions, --grounding)", + ) + args = parser.parse_args(argv) + if args.partial and not args.early: + parser.error("--partial requires --early") + if args.early and args.step != "prepare": + parser.error("--early only applies to prepare") + if args.step in ("prepare", "pin-inputs") and not args.runs_root: + parser.error(f"{args.step} requires --runs-root") + if args.step == "install-references": + missing = [ + f"--{name}" + for name in ("built", "explanations", "actions", "grounding") + if getattr(args, name) is None + ] + if missing: + parser.error(f"install-references requires {' '.join(missing)}") + for name in ("built", "explanations", "actions", "grounding"): + setattr(args, name, getattr(args, name).resolve()) + if args.step != "pin-inputs": + if args.stage_dir is None: + parser.error(f"{args.step} requires --stage-dir") + args.stage_dir = args.stage_dir.resolve() + return args + + +def main(argv=None) -> None: + args = parse_args(argv) + if args.step == "pin-inputs": + pin_inputs(args) + return + sources = [SNAPSHOT, ANNOTATIONS, args.base_predictions, args.base_payload] + sources += [ + p + for p in ( + args.runs_root, + args.audit_seed, + args.grounding, + args.built, + args.explanations, + args.actions, + ) + if p is not None + ] + validate_stage_path(args.stage_dir, sources) + stage = args.stage_dir + bundle = stage / "publish" / RUN_NAME + receipt_path = stage / "stage.json" + if args.step == "prepare": + runs = discover_new_models(args.runs_root) + print("All copied run states qualify; resolving 20261010 base", flush=True) + base, reference, live = resolve_base(args) + print("Base verified; folding copied additions", flush=True) + stage.mkdir(parents=True, exist_ok=True) + require( + not receipt_path.exists(), + "stage already prepared; use a fresh stage-dir " + "or resume judge/triage/export", + ) + bundle, frames = prepare_inputs(args, runs, base, reference) + if args.partial: + partial_scores(args, base, reference, frames) + seed = None + if args.early: + export(args, bundle, live) + else: + seed = prepare_cases(args, bundle) + inputs = [p for p in (stage / "inputs").rglob("*") if p.is_file()] + inputs += [ + bundle / "us" / name for name in (*REFERENCE_FILES, "predictions.csv") + ] + receipt = { + "partial": args.partial, + "early": args.early, + "base_tag": BASE_TAG, + "base_commit": BASE_COMMIT, + "files": {str(p.relative_to(stage)): digest(p) for p in inputs}, + } + if seed is not None: + receipt["seed"] = seed + write_json(receipt_path, receipt) + else: + receipt = json.loads(receipt_path.read_text()) + require( + not receipt["partial"] and not receipt["early"], + "early/partial stage cannot become a release; prepare fresh full inputs", + ) + for name, expected in receipt["files"].items(): + require( + digest(stage / name) == expected, + f"staged input changed: {name}; prepare a new stage", + ) + (stage / "release-ready.json").unlink(missing_ok=True) + if args.step in ("triage", "export", "adjudicate-exclusions"): + installed = receipt.get("exclusions_installed") or {} + if receipt.get("references_installed"): + # The release's record is the installed build's, re-gated here. + expected = verify_installed_references(stage, receipt).sha256[ + EXCLUSIONS_NAME + ] + else: + expected = release_exclusions_sha256() + require( + installed.get("sha256") == expected, + "the stage's exclusions are not the release's; run " + "--step install-exclusions", + ) + if args.step == "bind-seed": + bind_seed(args) + elif args.step == "install-references": + install_references(args) + elif args.step == "install-exclusions": + install_exclusions(args) + elif args.step == "adjudicate-exclusions": + adjudicate_exclusions(args) + elif args.step == "provenance": + write_judge_provenance(args) + elif args.step == "judge": + judge(args, bundle) + elif args.step == "triage": + triage(args, bundle) + else: + triage(args, bundle) + export(args, bundle, resolve_live_base(args)) + + +if __name__ == "__main__": + main() diff --git a/scripts/freeze_musewonder.py b/scripts/freeze_musewonder.py new file mode 100644 index 00000000..cb55e3ac --- /dev/null +++ b/scripts/freeze_musewonder.py @@ -0,0 +1,1641 @@ +"""Build the Claude Haiku 5.5 release locally from a strictly gated stage. + +Adapted from freeze_gpt61sol.py. It configures the existing freezer in process +and never calls a release upload. Run only after finish_haiku55.py --step +export has written release-ready.json. The optional --dry-run validates inputs +without changing repository files. Both rebuild the payload from the bound +bundle, as export builds it, and refuse a staged payload that differs. + +The release folds one model onto release 20261006 and installs the ten +exclusions Max ruled on 2026-10-06 (docs/haiku55/spec.json). Without an engine +upgrade, references stay release 20261006's and the exclusion record is that +release's plus the ten ruled records. An installed upgrade must still pass +the driver's build and drop-log gates; its references, explanations and +exclusions are frozen byte for byte. Scenarios always stay 20261006's. The +adjudications may change only through a staged record that triage applied and +export bound: the ruled outputs' entries, a re-opened case's judge fields, and +date conventions that name the 2026-10-06 wave. The committed wording +amendments record stays release 20261006's, with any stage list appended. +""" + +from __future__ import annotations + +import argparse +import copy +import datetime +import functools +import hashlib +import json +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parents[1] +os.environ["OPENBLAS_NUM_THREADS"] = "1" +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "scripts")) + +import finish_haiku55 as driver # noqa: E402 + +# The treatment check is release-independent; reuse it unchanged. +from freeze_adds0928 import validate_treatment # noqa: E402 + +# Reuse release 20261006's manifest comparison and baseline version counts. +from release_20261006 import leaf_paths # noqa: E402 +from release_20261006 import versions_description as base_count_exclusions # noqa: E402 + +RUN = driver.RUN_NAME +NEW_MODELS = dict(driver.MODELS) +BOARD_MODELS = driver.BOARD_MODELS +ADJUDICATIONS = "us_adjudications.json" +# The published annotations the adjudications and amendments are applied to. +ANNOTATION_CSVS = ("us_audit_row_annotations.csv", "us_case_notes.csv") +# The reference narratives a judge's prompt carries: release 20261006's, +# unless an installed engine upgrade regenerates them. +EXPLANATIONS = "us_case_reference_explanations.csv" +# The wording amendments record, as it is committed beside the adjudications. +COMMITTED_AMENDMENTS = f"us_{driver.AMENDMENTS}" +EXCLUSIONS = "reference_exclusions.json" +# The references pinned for a stage without an engine upgrade. +PINNED_REFERENCES = tuple( + name for name in driver.BASE_REFERENCE_SHA256 if name != EXCLUSIONS +) +SNAPSHOT = Path("paper/snapshot/20260501") +ANNOTATIONS = Path("annotations") / RUN +POINTER = Path("app/src/data.artifact.json") +VERSIONS = Path("app/src/data.versions.json") +# The engines the exclusion records were decided on, which the live version's +# description counts by name. +EXCLUSION_ENGINES = frozenset({"policyengine-us 1.755.4", "policyengine-us 2.15.17"}) +# Manifest paths the freeze may change from release 20261006's; every other +# leaf must equal that release's manifest, read from git at BASE_COMMIT. +MANIFEST_CHANGES = ( + ("audit_annotation_artifacts", "files"), + ("audit_annotation_artifacts", "developer_adjudications"), + ("audit_annotation_artifacts", "judge_provenance"), + # The note counts the audited prediction rows, which the addition extends. + ("audit_annotation_artifacts", "note"), + ("committed_snapshot_artifacts", "model_serving_config.json"), + ("committed_snapshot_artifacts", "us_impact_summary_by_model.csv"), + ("description",), + ("files",), + ("live_dashboard_artifact",), + ("model_response_date",), + ("published_dashboard_artifact",), + ("reference_exclusions",), + ("reference_output_refresh", "snapshot_date"), + ("rendered_paper_artifacts",), + ("reproducibility_notes",), + ("scope", "models"), + ("snapshot_date",), + ("source_run_artifacts", RUN, "files"), +) +# Pins under those paths that must stay release 20261006's all the same. +MANIFEST_PINNED = frozenset( + { + *(("source_run_artifacts", RUN, "files", name) for name in PINNED_REFERENCES), + ("audit_annotation_artifacts", "files", EXPLANATIONS), + } +) +# Additional leaves written from an installed build's sidecar. Each is named +# explicitly; the scenario pins, household dataset and other refresh fields +# remain release 20261006's, including row_count and generated_at_utc. +UPGRADE_MANIFEST_CHANGES = ( + ("committed_snapshot_artifacts", "us_reference_outputs.csv"), + ("reference_output_refresh", "date"), + ("reference_output_refresh", "regenerated_at_utc"), + ("reference_output_refresh", "reference_csv_sha256"), + ("reference_output_refresh", "policyengine_version"), + ("reference_output_refresh", "policyengine_us_version"), + ("reference_output_refresh", "policyengine_us_data_build_id"), + ("reference_output_refresh", "policyengine_us_dataset"), + ("reference_output_refresh", "policyengine_us_dataset_uri"), + ("reference_output_refresh", "policyengine_us_data_artifact_sha256"), +) +UPGRADE_MANIFEST_UNPINNED = frozenset( + { + ("source_run_artifacts", RUN, "files", "reference_outputs.csv"), + ("source_run_artifacts", RUN, "files", "reference_outputs.csv.meta.json"), + ("audit_annotation_artifacts", "files", EXPLANATIONS), + } +) + + +def read_json(path: Path) -> dict: + """Read an object, with the filename retained in parse failures.""" + return json.loads(path.read_text()) + + +def write_json(path: Path, value: dict) -> None: + """Write reviewable deterministic JSON.""" + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def digest(path: Path) -> str: + """Hash a file without holding its bytes in memory.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def judged_cases(audit: Path) -> list[str]: + """The cases validate_verdicts requires a verdict for, from the manifest. + + A stage without a manifest has none; the receipt then fails to bind + audit/cases.jsonl. + """ + manifest = audit / "cases.jsonl" + if not manifest.is_file(): + return [] + items = map(json.loads, manifest.read_text().splitlines()) + return [item["case_id"] for item in items if not item["parse_failure_only"]] + + +def verify_freezer_destinations(freezer) -> None: + """freeze_snapshot must write into this checkout, and only there. + + Both modules find the checkout from their own file, so they agree unless + another checkout's freeze_snapshot.py comes first on the import path; that + freezer would overwrite the other checkout's tracked snapshot. + """ + expected = { + "ROOT": ROOT, + "SNAPSHOT_DIR": ROOT / SNAPSHOT, + "RUN_DEST": ROOT / SNAPSHOT / "runs" / RUN, + "ANNOTATIONS_DEST": ROOT / ANNOTATIONS, + } + elsewhere = { + name: str(getattr(freezer, name)) + for name, path in expected.items() + if Path(getattr(freezer, name)).resolve() != path.resolve() + } + if elsewhere or freezer.RUN_LABEL != RUN: + raise SystemExit( + f"freeze_snapshot would write outside this checkout ({ROOT}): " + f"{elsewhere or freezer.RUN_LABEL}" + ) + + +def release_spec(tag: str) -> dict: + """docs/haiku55/spec.json, with what the freeze reads from it checked. + + The snapshot date and the day the 2026-10-06 wave's decisions were written + come from the spec alone, so a spec that lacks either stops the freeze + here, before any gate reads it. The spec must also name the release this + freeze builds and the base the driver pins. + """ + spec = driver.load_spec() + try: + datetime.date.fromisoformat(spec.get("snapshot_date")) + except (TypeError, ValueError): + raise SystemExit( + f"{driver.SPEC_PATH} names no snapshot_date (YYYY-MM-DD): " + f"{spec.get('snapshot_date')!r}" + ) from None + written = spec.get("adjudications_written_on") + if not isinstance(written, str) or not written.strip(): + raise SystemExit( + f"{driver.SPEC_PATH} names no adjudications_written_on, the UTC day " + "the 2026-10-06 wave's decisions were written; the record's date " + "conventions state it" + ) + named = { + "release_tag": tag, + "base_tag": driver.BASE_TAG, + "base_sha256": driver.BASE_SHA256, + } + differ = { + field: spec.get(field) + for field, value in named.items() + if spec.get(field) != value + } + if differ: + raise SystemExit( + f"{driver.SPEC_PATH} names another release or base than this freeze " + f"builds ({named}): {differ}" + ) + return spec + + +def engine_upgrade_installed(stage: Path) -> bool: + """Whether the stage's stage.json records an installed engine upgrade of + the references; an unreadable stage.json is left to the later gates.""" + try: + receipt = read_json(stage / "stage.json") + except (OSError, ValueError): + return False + return isinstance(receipt, dict) and bool(receipt.get("references_installed")) + + +def verify_upgrade(stage: Path, receipt: dict) -> driver.Upgrade | None: + """Re-gate the installed build and drops, all bound by strict export. + + The driver checks the build against git and the spec, then every installed + copy. The export must bind every retained build file independently of + stage.json. A build adding exclusions freezes only when the spec's + upgrade_adjudications decide exactly those outputs (driver.upgrade_decisions). + """ + prepared = read_json(stage / "stage.json") + installed = prepared.get("references_installed") + if not installed: + return None + if not isinstance(installed, dict): + raise SystemExit("stage.json's references_installed is not an installed build") + required = [str(Path(driver.BUILD_COPY) / n) for n in driver.BUILD_COPY_FILES] + unbound = [name for name in required if name not in receipt["files"]] + if unbound: + raise SystemExit( + f"Strict export receipt does not bind the installed build: {unbound}" + ) + upgrade = driver.verify_installed_references(stage, prepared) + for name in ("scenarios.csv", "scenarios.csv.meta.json"): + path = stage / "scoring" / name + if not path.is_file() or digest(path) != driver.BASE_REFERENCE_SHA256[name]: + raise SystemExit(f"Staged scoring/{name} is not release 20261006's") + superseded = installed.get("superseded") + if ( + not isinstance(superseded, list) + or not all(isinstance(item, dict) for item in superseded) + or driver.references_record(upgrade, installed) != installed + ): + raise SystemExit( + "stage.json's references_installed is not the installed build's record" + ) + wrong = [ + name + for name in driver.BUILD_COPY_FILES + if receipt["files"][str(Path(driver.BUILD_COPY) / name)] != upgrade.sha256[name] + ] + if wrong: + raise SystemExit( + f"Strict export receipt does not bind the installed build's bytes: {wrong}" + ) + driver.verify_recorded_drops(stage, upgrade) + # The outputs it newly excludes are decided by the spec's + # upgrade_adjudications, which verify_adjudication_record holds exact. + driver.upgrade_decisions(upgrade) + return upgrade + + +def verify_verdicts(stage: Path) -> None: + """Every judged verdict passes the driver's own gate against the bound seed. + + Export validates verdicts before it writes the receipt, but the receipt + only binds hashes: a kept verdict edited afterwards, its sidecar and + receipt entry re-hashed, must still be the seed's byte for byte, and every + new one bound to its prompt and its Opus 5.5 judge. Without + ``remove_invalid`` validate_verdicts changes nothing. + """ + pending = driver.validate_verdicts(stage / "audit", seed=driver.load_seed(stage)) + if pending: + raise SystemExit( + f"{len(pending)} staged verdicts fail validation: {pending[:8]}; " + "run judge and export again" + ) + + +def verify_receipt(stage: Path, payload_path: Path, tag: str) -> dict: + """The strict export receipt must bind this payload and all staged evidence.""" + receipt = read_json(stage / "release-ready.json") + if ( + receipt.get("payload_sha256") != digest(payload_path) + or receipt.get("models") != BOARD_MODELS + or receipt.get("partial") is not False + or receipt.get("release_tag") != tag + ): + raise SystemExit( + f"Strict export receipt does not match {payload_path.name} " + f"(release tag {tag!r}; re-export after changing RELEASE_TAG)" + ) + if ( + receipt.get("base_tag") != driver.BASE_TAG + or receipt.get("base_sha256") != driver.BASE_SHA256 + ): + raise SystemExit( + "Strict export receipt does not name release 20261006 as its base " + f"({receipt.get('base_tag')!r}, {receipt.get('base_sha256')!r})" + ) + if not receipt.get("files"): + raise SystemExit("Strict export receipt is missing staged evidence hashes") + for name, expected in receipt["files"].items(): + source = (stage / name).resolve() + if ( + not source.is_relative_to(stage) + or not source.is_file() + or digest(source) != expected + ): + raise SystemExit(f"Staged evidence changed since strict export: {name}") + bundle = Path("publish") / RUN + required = [bundle / "us" / name for name in driver.REFERENCE_FILES] + required += [bundle / "us/predictions.csv", bundle / "annotations" / ADJUDICATIONS] + required += [bundle / "annotations" / name for name in ANNOTATION_CSVS] + required += [bundle / "annotations" / EXPLANATIONS] + # Each new model's pinned run files, and the prepare-time hashes. + required += [ + Path("inputs") / slug / name + for slug in NEW_MODELS + for name in driver.PINNED_INPUTS + ] + required += [Path("model-provenance.json")] + # What the adjudication and verdict gates allow rests on these; stage.json + # also records the exclusions install-exclusions wrote. + required += [Path(driver.PROMPT_CHANGES), Path("stage.json")] + if (stage / driver.AMENDMENTS).exists(): + required.append(Path(driver.AMENDMENTS)) + # Every judged case's audit evidence, as validate_verdicts reads it: an + # unbound verdict or sidecar could be edited and re-hashed after export. + # So could the transcript the provenance gate reads for an isolated judge. + required += [Path("audit/cases.jsonl"), Path("audit/schema.json")] + for case in judged_cases(stage / "audit"): + directory = Path("audit/cases") / case + required += [ + directory / name for name in ("verdict.json", "verdict.meta.json") + ] + [directory / "prompt.md"] + if (stage / directory / "claude.transcript.jsonl").is_file(): + required.append(directory / "claude.transcript.jsonl") + unbound = [str(p) for p in required if str(p) not in receipt["files"]] + if unbound: + raise SystemExit(f"Strict export receipt does not bind: {unbound}") + return receipt + + +def verify_installed_exclusions( + stage: Path, receipt: dict, text: str, upgrade: driver.Upgrade | None = None +) -> str: + """The stage scores on the release's exclusion record, as installed. + + ``text`` is release 20261006's record plus the ten ruled records + (finish_haiku55.build_release_exclusions, spelled by exclusions_text), + rebuilt now from git and the spec. The staged record must be those bytes, + the receipt must bind them, and stage.json, which the receipt binds, must + record them as --step install-exclusions wrote them: the installed and + base hashes, the record count, and the prepared file's hash. Returns the + record's sha256. + """ + bound = str(Path("publish") / RUN / "us" / EXCLUSIONS) + expected = hashlib.sha256(text.encode()).hexdigest() + staged = stage / bound + if not staged.is_file() or staged.read_bytes() != text.encode(): + if upgrade is not None: + raise SystemExit( + f"Staged {EXCLUSIONS} is not the installed engine upgrade's record" + ) + raise SystemExit( + f"Staged {EXCLUSIONS} is not release 20261006's record plus the " + f"{driver.NEW_EXCLUSIONS} ruled records ({driver.SPEC_PATH}); run " + "--step install-exclusions, then triage and export again" + ) + if receipt["files"].get(bound) != expected: + raise SystemExit( + f"Strict export receipt does not bind the installed {EXCLUSIONS}" + ) + prepared = read_json(stage / "stage.json") + installed = prepared.get("exclusions_installed") + recorded = prepared.get("files") + records = driver.RELEASE_EXCLUSIONS if upgrade is None else upgrade.records + if ( + not isinstance(installed, dict) + or not isinstance(recorded, dict) + or installed.get("sha256") != expected + or installed.get("base_sha256") != driver.BASE_REFERENCE_SHA256[EXCLUSIONS] + or installed.get("records") != records + or recorded.get(bound) != expected + or ( + upgrade is not None + and ( + installed.get("source") != f"{driver.BUILD_COPY}/{EXCLUSIONS}" + or installed.get("engine_version") != upgrade.engine_version + or installed.get("spec_sha256") + != digest(driver.ROOT / driver.SPEC_PATH) + or expected != upgrade.sha256[EXCLUSIONS] + or text != upgrade.exclusions_text + ) + ) + ): + step = "install-exclusions" if upgrade is None else "install-references" + raise SystemExit( + f"stage.json does not record the release's {records} " + f"exclusions as --step {step} writes them " + f"(exclusions_installed {installed!r}); run it, then triage and " + "export again" + ) + return expected + + +def verify_scored_outputs( + country: dict, exclusions: list[dict], upgrade: driver.Upgrade | None = None +) -> None: + """The payload publishes the release's exclusions and scores what remains. + + Its referenceExclusions name the record's outputs, in the record's order, + and every model is scored on the outputs left. The rebuild checks the + whole payload; this names the two counts the release states. + """ + published = [ + (item.get("scenarioId"), item.get("variable")) + for item in country.get("referenceExclusions", []) + ] + expected = [driver.spec_key(record) for record in exclusions] + if upgrade is not None and len(expected) != upgrade.records: + raise SystemExit( + "The payload's exclusion record does not count the installed " + "build's outputs" + ) + if published != expected: + differ = sorted(set(published) ^ set(expected)) or ["their order"] + raise SystemExit( + f"The payload's {len(published)} excluded outputs are not the " + f"release's {len(expected)}: {differ[:8]}" + ) + scored = {row["model"]: row.get("n") for row in country["modelStats"]} + count = driver.RELEASE_SCORED if upgrade is None else upgrade.scored_outputs + off = {model: n for model, n in scored.items() if n != count} + if off: + raise SystemExit(f"Every model must be scored on {count} outputs: {off}") + + +def verify_judge_provenance( + stage: Path, receipt: dict, rejudged: frozenset[str] +) -> None: + """The judge provenance record is the one export bound, and still true. + + The record is a repository file (driver.JUDGE_PROVENANCE), so the receipt + binds it apart from the stage's files. Its bytes must be the ones export + hashed, and it must still describe the staged new verdicts and their + transcripts, as export checked. + """ + bound = receipt.get("judge_provenance") + if ( + not isinstance(bound, dict) + or bound.get("path") != driver.JUDGE_PROVENANCE_PATH + or not driver.JUDGE_PROVENANCE.is_file() + or bound.get("sha256") != digest(driver.JUDGE_PROVENANCE) + ): + raise SystemExit( + f"{driver.JUDGE_PROVENANCE_PATH} is not the judge provenance record " + "the strict export bound; export again" + ) + driver.verify_judge_provenance( + stage / "audit" / "cases", rejudged, driver.JUDGE_PROVENANCE + ) + + +def verify_references( + source_us: Path, + frozen_run: Path, + manifest: dict, + exclusions_sha256: str, + upgrade: driver.Upgrade | None = None, +) -> None: + """References match the baseline or the gated installed upgrade's pins. + + The reference outputs, the scenarios and their sidecars must equal release + 20261006's pins in the stage, in the committed snapshot and in the + manifest. The staged exclusion record must be the release's + (``exclusions_sha256``). Its committed copy and its manifest pin are + release 20261006's before this freeze and the release's after it, so a + second freeze finds either. + With an upgrade, the build's three reference files replace the staged + pins, while committed copies and manifest pins may be baseline or build. + Scenarios and their sidecar remain pinned to release 20261006. + """ + pins = manifest["source_run_artifacts"][RUN]["files"] + upgraded = {} if upgrade is None else driver.upgraded_reference_pins(upgrade) + + def matches(path: Path, allowed: set[str]) -> bool: + return path.is_file() and digest(path) in allowed + + for name, pin in driver.BASE_REFERENCE_SHA256.items(): + staged, committed = {pin}, {pin} + reason = "this release revises no reference value" + whose = "not release 20261006's" + if name == EXCLUSIONS: + staged, committed = {exclusions_sha256}, {pin, exclusions_sha256} + reason = "it is not the release's record" + whose = "neither release 20261006's nor this release's" + elif upgrade is not None and name in driver.UPGRADED_FILES: + staged, committed = {upgraded[name]}, {pin, upgraded[name]} + reason = "it is not the installed engine upgrade's file" + whose = "neither release 20261006's nor the installed engine upgrade's" + if not matches(source_us / name, staged): + raise SystemExit(f"Staged {name} changed: {reason}") + if not matches(frozen_run / name, committed): + raise SystemExit(f"Committed {name} changed: it is {whose}") + if pins.get(name) not in committed: + raise SystemExit(f"Manifest pin for {name} is {whose}") + if upgrade is not None: + allowed = { + driver.BASE_REFERENCE_SHA256[driver.CSV_NAME], + upgraded[driver.CSV_NAME], + } + if not matches(frozen_run.parents[1] / "us_reference_outputs.csv", allowed): + raise SystemExit( + "Committed us_reference_outputs.csv changed: it is neither " + "release 20261006's nor the installed engine upgrade's" + ) + if ( + manifest["committed_snapshot_artifacts"].get("us_reference_outputs.csv") + not in allowed + ): + raise SystemExit( + "Manifest pin for us_reference_outputs.csv is neither release " + "20261006's nor the installed engine upgrade's" + ) + if manifest["audit_annotation_artifacts"]["files"].get(EXPLANATIONS) not in { + driver.BASE_EXPLANATIONS_SHA256, + upgrade.sha256[EXPLANATIONS], + }: + raise SystemExit( + f"Manifest pin for {EXPLANATIONS} is neither release " + "20261006's nor the installed engine upgrade's" + ) + + +def verify_explanations( + staged_annotations: Path, + upgrade: driver.Upgrade | None = None, + *, + stage: Path | None = None, +) -> None: + """Explanations are baseline bytes or the installed build's exact bytes. + + A judge's prompt carries them, and this release rewords none: prepare + copies the committed file, and no step writes it. The payload publishes + them, so an explanation edited after export, with the payload rebuilt to + match and every hash updated, would otherwise pass. An installed upgrade + instead pins them to its retained explanations file, byte for byte. + """ + if upgrade is None: + base = driver.base_commit_blob(ANNOTATIONS / EXPLANATIONS) + else: + stage = staged_annotations.parents[2] if stage is None else stage + built = stage / driver.BUILD_COPY / EXPLANATIONS + if not built.is_file() or digest(built) != upgrade.sha256[EXPLANATIONS]: + raise SystemExit(f"The installed build's {EXPLANATIONS} changed") + base = built.read_bytes() + staged = staged_annotations / EXPLANATIONS + if not staged.is_file() or staged.read_bytes() != base: + if upgrade is not None: + raise SystemExit( + f"Staged {EXPLANATIONS} is not the installed engine upgrade's " + "file byte for byte" + ) + raise SystemExit( + f"Staged {EXPLANATIONS} changed: this release rewords no reference " + "explanation" + ) + + +def verify_adjudication_record( + staged: Path, + source_us: Path, + rejudged: frozenset[str], + amendments: list[dict], + cases_dir: Path, + upgrade: driver.Upgrade | None = None, +) -> int: + """The staged record changes release 20261006's only as the release says. + + The baseline is release 20261006's record in git (BASE_COMMIT), never the + working-tree copy this freeze overwrites. The ten ruled outputs must carry + exactly the entries the driver builds for them from the spec and each + case's bound verdict. A case Claude Haiku 5.5 re-opened may rewrite its + judge fields, restated as the restate script does, and its reasoning + exactly as the listed wording amendments say. Nothing else in any + committed entry may change. The record excludes exactly the outputs the + staged exclusion record lists. The staged file's bytes must be exactly its + parsed content, with release 20261006's note, and its date conventions + with the 2026-10-06 wave named (driver.release_date_conventions). Returns + how many staged decisions, beyond the ruled ones, the committed record + lacks. The staged record, prompt-changes.json and the amendments are bound + by the receipt, which export writes only after triage applied them. + """ + from policybench.adjudications import excluded_case_keys, load_adjudications + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + decisions = driver.upgrade_decisions(upgrade) + driver.verify_record_form( + staged.read_text(), + driver.base_adjudication_record(), + driver.upgrade_wave(decisions), + ) + after = load_adjudications(staged) + base = driver.base_adjudications() + if upgrade is None: + added = driver.verify_adjudication_changes( + base, after, rejudged, amendments, cases_dir + ) + dropped = frozenset() + else: + # The outputs the upgrade newly excludes, and the triage decisions on + # its re-opened cases, are the spec's, gated exactly as triage gates + # them (driver.stage_adjudications). + dropped = upgrade.dropped_cases + added = driver.verify_adjudication_changes( + base, + after, + rejudged, + amendments, + cases_dir, + regenerated=upgrade.regenerated_ruled, + dropped=dropped, + added=decisions, + triage=driver.triage_items(upgrade), + ) + # A dropped decision is gone, so an entry on its case is new, not restated. + driver.verify_restatements( + [entry for entry in base if driver.case_id(entry) not in dropped], + after, + rejudged, + cases_dir, + ) + decided = excluded_case_keys(after) + excluded = exclusion_keys(load_reference_exclusions(source_us)) + if decided != excluded: + count = driver.RELEASE_EXCLUSIONS if upgrade is None else upgrade.records + source = "the spec" if upgrade is None else "the installed build" + raise SystemExit( + "Staged adjudications and the staged exclusion record disagree on " + f"the excluded outputs: {sorted(decided ^ excluded)[:8]}; the release " + f"excludes the {count} {source} lists" + ) + return added + + +def payload_differences( + rebuilt, staged, path: str = "payload", limit: int = 5 +) -> list[str]: + """The first ``limit`` paths at which two parsed JSON values differ. + + Key order counts, as it does in the bytes: the paths are empty exactly + when the two values serialize to the same JSON. A list item that names a + model is labelled with it. + """ + found: list[str] = [] + + def walk(a, b, at: str) -> None: + if len(found) >= limit: + return + if isinstance(a, dict) and isinstance(b, dict): + for key in a: + if key in b: + walk(a[key], b[key], f"{at}.{key}") + else: + found.append(f"{at}.{key} (only rebuilt)") + found.extend(f"{at}.{key} (only staged)" for key in b if key not in a) + if a.keys() == b.keys() and list(a) != list(b): + found.append(f"{at} (key order)") + elif isinstance(a, list) and isinstance(b, list): + for index, (x, y) in enumerate(zip(a, b)): + name = x.get("model") if isinstance(x, dict) else None + label = f"{index} {name}" if isinstance(name, str) else index + walk(x, y, f"{at}[{label}]") + if len(a) != len(b): + found.append(f"{at} (length {len(a)} rebuilt, {len(b)} staged)") + elif json.dumps(a) != json.dumps(b): + found.append(at) + + walk(rebuilt, staged, path) + return found[:limit] + + +def rebuild_payload( + stage: Path, + receipt: dict, + payload_path: Path, + live: dict, + upgrade: driver.Upgrade | None = None, +) -> None: + """The staged payload must be what export builds from the bound bundle. + + The receipt binds the payload only by a hash that sits beside it, so a + payload edited after export (the addition's scores, an incumbent's cost, a + case's classes) with its receipt rehashed passes every other gate. + Export's own build (driver.build_payload, against release 20261006 as + ``live``) runs again on a scratch copy of the bundle files the receipt + binds, each checked against its receipt hash, and its bytes must equal the + staged payload's. That build scores every model on the release's outputs + and repeats export's scope check: with release 20261006's exclusion + record put back, every incumbent's modelStats is that release's, byte for + byte. The copy is the bundle's: export_full_run writes data.json, + us/data.json and us/analysis/ into the bundle it reads, and its data.json + lacks the carried usage, so a rebuild in place would rewrite the stage. + """ + bundle = Path("publish") / RUN + with tempfile.TemporaryDirectory(prefix="freeze-haiku55-") as scratch: + copy_root = Path(scratch) / RUN + for name, expected in receipt["files"].items(): + if not Path(name).is_relative_to(bundle): + continue + target = copy_root / Path(name).relative_to(bundle) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(stage / name, target) + if digest(target) != expected: + raise SystemExit(f"Staged evidence changed since strict export: {name}") + options = {} if upgrade is None else {"upgrade": upgrade} + rebuilt = driver.payload_text(driver.build_payload(copy_root, live, **options)) + staged = payload_path.read_bytes() + if rebuilt.encode() != staged: + differ = payload_differences(json.loads(rebuilt), json.loads(staged)) + raise SystemExit( + f"Staged {payload_path.name} is not what export builds from the " + f"bound bundle; it differs at {differ or ['its serialization']}; " + "export again" + ) + + +def base_prediction_rows(): + """Release 20261006's predictions, read from git at BASE_COMMIT. + + Never the working-tree copy, which the freezer rewrites from the stage: a + freeze that stopped partway would compare the staged rows with themselves. + """ + import io + + import pandas as pd + + path = SNAPSHOT / "runs" / RUN / "predictions.csv.gz" + return pd.read_csv( + io.BytesIO(driver.base_commit_blob(path)), compression="gzip", low_memory=False + ) + + +def base_serving_config() -> dict: + """Release 20261006's serving configuration, read from git at BASE_COMMIT.""" + return json.loads(driver.base_commit_blob(SNAPSHOT / "model_serving_config.json")) + + +def annotation_differences( + rebuilt_text: str, staged: Path, key: list[str], limit: int = 4 +) -> list[str]: + """The first ``limit`` places where a staged annotation CSV is not the + rebuilt one: a column set or order, a row only one side has, or a cell + (named by its case, model and column). Empty when the two parse to the + same cells, so only the row order or the serialization differs.""" + import io + + import pandas as pd + + def read(source) -> pd.DataFrame: + return pd.read_csv(source, dtype=str, keep_default_na=False) + + try: + rebuilt, published = read(io.StringIO(rebuilt_text)), read(staged) + except ValueError: + return ["its serialization"] + if list(rebuilt.columns) != list(published.columns): + return [f"its columns {list(published.columns)}, not {list(rebuilt.columns)}"] + + def by_key(frame: pd.DataFrame) -> dict[tuple, dict]: + return {tuple(row[k] for k in key): row for row in frame.to_dict("records")} + + found = [] + if published.duplicated(key).any(): + found.append(f"duplicate {key} rows") + built, staged_rows = by_key(rebuilt), by_key(published) + for item in sorted(set(built) | set(staged_rows)): + name = "__".join(item) + if item not in staged_rows or item not in built: + side = "rebuilt" if item in built else "staged" + found.append(f"{name} (only {side})") + continue + found += [ + f"{name} {column}" + for column in rebuilt.columns + if built[item][column] != staged_rows[item][column] + ] + return found[:limit] + + +def verify_annotation_amendments( + annotations: Path, amendments: list[dict], audit: Path +) -> None: + """The staged row annotations and case notes are what triage builds from + the verdicts, the staged adjudications and the listed amendments, in every + column. + + Both frames are rebuilt as triage builds them: collect_audit folds each + verdict into its rows (the model's failure class and diagnosis) and its + case note (the case's classes, flag, hypothesis and rationale); the staged + adjudication record, whose reasoning verify_adjudication_record has + already replayed, sets each decided case's classes and appends its + sentence to the case note; then the listed amendments are replayed in + order, each old wording found exactly once when it is applied. Each staged + file must be the rebuilt frame's bytes as triage writes them + (driver.annotation_csv_text), so a failure class, flag or count edited on + a case no adjudication decides is refused as surely as unlisted wording, + even beside a listed fragment. + """ + from policybench.adjudications import apply_adjudications, load_adjudications + from policybench.audit import collect_audit + + collected = collect_audit(annotations.parent / "us", audit) + rows, cases, _ = apply_adjudications( + collected["row"], + collected["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ), + load_adjudications(annotations / ADJUDICATIONS), + ) + driver.amend_annotations(rows, cases, amendments) + for name, rebuilt, key in ( + ("us_case_notes.csv", cases, driver.KEY), + ("us_audit_row_annotations.csv", rows, [*driver.KEY, "model"]), + ): + text = driver.annotation_csv_text(rebuilt) + if text.encode() != (annotations / name).read_bytes(): + differ = annotation_differences(text, annotations / name, ["country", *key]) + raise SystemExit( + f"Staged {name} is not what triage builds from the verdicts, the " + "staged adjudications and the listed wording amendments; it " + f"differs at {differ or ['its row order or serialization']}; run " + "triage and export again" + ) + + +def amendments_text(record: dict) -> str: + """The wording amendments record as the committed file spells it.""" + return json.dumps(record, indent=1, ensure_ascii=False) + + +def committed_amendments(stage: Path) -> bytes: + """The wording amendments record this freeze commits. + + Release 20261006's committed record (from git at BASE_COMMIT) lists the + amendments GPT-6.1 Sol's re-opened cases needed, and that wording is still + published, so the record is never replaced. With no stage list, or an + empty one, it is committed unchanged. A stage list (driver.load_amendments + checks each item names a case Claude Haiku 5.5 re-opened) is appended + after the earlier amendments, in its own order, and the note says which + items are this release's; a note the stage list carries follows it. + """ + base = driver.base_commit_blob(ANNOTATIONS / COMMITTED_AMENDMENTS) + source = stage / driver.AMENDMENTS + listed = json.loads(source.read_text()) if source.exists() else {} + added = listed.get("amendments") or [] + if not added: + return base + record = json.loads(base) + if amendments_text(record).encode() != base or list(record) != [ + "note", + "amendments", + ]: + raise SystemExit( + f"Release 20261006's {COMMITTED_AMENDMENTS} is not in its committed " + "form; cannot append this release's wording amendments" + ) + earlier = len(record["amendments"]) + note = ( + f"{record['note']} The last {len(added)} (after the first {earlier}) " + "are for cases Claude Haiku 5.5 re-opened (finish_haiku55.py " + "load_amendments)." + ) + if isinstance(listed.get("note"), str) and listed["note"].strip(): + note += " " + listed["note"].strip() + merged = {"note": note, "amendments": record["amendments"] + added} + return amendments_text(merged).encode() + + +def freeze_amendments(stage: Path, destination: Path) -> None: + """Commit the wording amendments record beside the adjudication record. + + freeze_snapshot rebuilds the annotations directory from the stage and so + removes the committed record; this writes it back (committed_amendments). + """ + (destination / COMMITTED_AMENDMENTS).write_bytes(committed_amendments(stage)) + + +def evidence_runs() -> dict[str, str]: + """The supervised run each new model's frozen serving evidence names. + + Its runs root and slug, read from the input pins as committed at HEAD + (``run`` there is ``//run``): the same record the staged + run files are checked against. + """ + record = json.loads(driver.head_blob(driver.INPUT_PINS_PATH)) + runs = {} + for slug in NEW_MODELS: + named = (record.get("inputs") or {}).get(slug, {}).get("run") + run = PurePosixPath(named or "") + if len(run.parts) != 3 or run.parts[1:] != (slug, "run"): + raise SystemExit( + f"{driver.INPUT_PINS_PATH} does not name the supervised run of " + f"{slug} as /{slug}/run: {named!r}" + ) + runs[slug] = run.parent.as_posix() + return runs + + +def serving_problems( + serving: dict, previous: dict, incumbents: set[str], runs: dict[str, str] +) -> list[str]: + """Where a serving configuration is not the release's. + + Its models are the incumbents and the addition. Every incumbent's row is + release 20261006's (``previous``), every field. Each new model's evidence + is its run state and names its supervised run. The evidence summary is the + tally of the rows' evidence kinds. + """ + models = serving.get("models") + if not isinstance(models, dict): + return ["no models"] + problems = [] + expected = set(incumbents) | set(NEW_MODELS.values()) + if set(models) != expected: + problems.append(f"roster differs on {sorted(set(models) ^ expected)[:8]}") + problems += [ + f"{model} is not release 20261006's row" + for model in sorted(set(incumbents) & set(models)) + if json.dumps(models[model], sort_keys=True) + != json.dumps(previous["models"][model], sort_keys=True) + ] + for slug, model in NEW_MODELS.items(): + evidence = (models.get(model) or {}).get("evidence") or {} + if evidence.get("kind") != "run_state": + problems.append(f"{model} has no run-state evidence") + elif evidence.get("run") != runs[slug]: + problems.append( + f"{model}'s evidence names run {evidence.get('run')!r}, not " + f"{runs[slug]!r}" + ) + counts = {"run_state": 0, "registry": 0} + for treatment in models.values(): + kind = (treatment.get("evidence") or {}).get("kind") + counts[kind] = counts.get(kind, 0) + 1 + if serving.get("evidence_summary") != counts: + problems.append( + f"evidence_summary {serving.get('evidence_summary')!r} is not the " + f"rows' tally {counts!r}" + ) + return problems + + +def merge_serving( + serving: dict, previous: dict, incumbents: set[str], runs: dict[str, str] +) -> dict: + """The release's serving configuration, from the freezer's (``serving``). + + The old rows already have frozen evidence. They are release 20261006's, + kept field for field after the incumbents' predictions were checked + identical, instead of being silently downgraded when this checkout lacks + the historical run directories. Each new model keeps the freezer's row, + whose evidence must be its run state and is made to name its supervised + run. The evidence summary is recounted. + """ + merged = copy.deepcopy(serving) + models = merged["models"] + for model in incumbents: + models[model] = copy.deepcopy(previous["models"][model]) + for slug, model in NEW_MODELS.items(): + evidence = models[model]["evidence"] + if evidence["kind"] != "run_state": + raise SystemExit(f"Missing frozen run-state evidence for {model}") + evidence["run"] = runs[slug] + counts = {"run_state": 0, "registry": 0} + for treatment in models.values(): + counts[treatment["evidence"]["kind"]] += 1 + merged["evidence_summary"] = counts + problems = serving_problems(merged, previous, incumbents, runs) + if problems: + raise SystemExit(f"Frozen serving configuration: {problems[:8]}") + return merged + + +def release_pointer(tag: str, payload_path: Path, payload_hash: str) -> dict: + """The dashboard pointer the freeze commits: the staged payload as the + release asset dashboard-data.json under ``tag``.""" + return { + "version": 1, + "repo": "PolicyEngine/policybench", + "tag": tag, + "asset": "dashboard-data.json", + "url": ( + "https://github.com/PolicyEngine/policybench/releases/download/" + f"{tag}/dashboard-data.json" + ), + "sha256": payload_hash, + "bytes": payload_path.stat().st_size, + } + + +def count_exclusions( + text: str, exclusions: list[dict], upgrade: driver.Upgrade | None = None +) -> str: + """Count the record's engines, including the installed build's engine.""" + if upgrade is None: + return base_count_exclusions(text, exclusions) + engines = (*sorted(EXCLUSION_ENGINES), upgrade.engine_version) + version = upgrade.engine_version.removeprefix("policyengine-us ") + counts = { + engine: sum(r["engine_version"] == engine for r in exclusions) + for engine in engines + } + pattern = ( + r"the \d+ excluded outputs keep the values they were decided on: \d+ from " + r"policyengine-us 1\.755\.4, \d+ from 2\.15\.17" + rf"(?:, \d+ from {re.escape(version)})?" + ) + if len(re.findall(pattern, text)) != 1: + raise SystemExit("the version description changed form") + described = [ + f"{counts[engine]} from " + + (engine if i == 0 else engine.removeprefix("policyengine-us ")) + for i, engine in enumerate(engines) + if i < len(EXCLUSION_ENGINES) or counts[engine] + ] + counted = ( + f"the {len(exclusions)} excluded outputs keep the values they were decided on: " + + ", ".join(described) + ) + text = re.sub(pattern, counted, text) + previous = driver.BASE_ENGINE.removeprefix("policyengine-us ") + versions = rf"(?:{re.escape(previous)}|{re.escape(version)})" + text, changed = re.subn( + rf"^Scored reference outputs from policyengine-us {versions} \(", + f"Scored reference outputs from {upgrade.engine_version} (", + text, + ) + if changed != 1: + raise SystemExit("the version description names an unexpected reference engine") + rechecked = len(upgrade.revision.get("excluded_outputs_rechecked", [])) + clause = ( + f", and the {rechecked} that move on {version} were re-reviewed " + "and stay excluded" + if rechecked + else "" + ) + text, changed = re.subn( + rf"({re.escape(counted)})(?:, and the \d+ that move on {versions} " + r"were re-reviewed " + r"and stay excluded)?\); same household facts", + lambda match: match[1] + clause + "); same household facts", + text, + ) + if changed != 1: + raise SystemExit("the version description has an unexpected re-review shape") + return text + + +def release_versions( + versions: dict, + exclusions: list[dict], + snapshot_date: str, + upgrade: driver.Upgrade | None = None, +) -> dict: + """The version list with the live version describing this release. + + The live version is the default one, and its artifact is the live + pointer. Its description counts the excluded outputs by the engine each + record was decided on and ends with the board's model count; its snapshot + label carries the spec's snapshot date. Nothing else changes, and applying + this to its own result changes nothing. + """ + out = copy.deepcopy(versions) + live = [item for item in out["versions"] if item["id"] == out["default"]] + if len(live) != 1 or live[0].get("artifact") != {"pointer": "live"}: + raise SystemExit("The default version is not the one live-pointer version") + engines = {record["engine_version"] for record in exclusions} + allowed = ( + EXCLUSION_ENGINES + if upgrade is None + else EXCLUSION_ENGINES | {upgrade.engine_version} + ) + if (upgrade is None and engines != allowed) or not engines <= allowed: + raise SystemExit( + "The live version description counts exclusions decided on " + f"{sorted(allowed)}; the record's are {sorted(engines)}" + ) + description, changed = re.subn( + r" - \d+ models$", + f" - {BOARD_MODELS} models", + count_exclusions(live[0]["description"], exclusions, upgrade), + ) + if changed != 1: + raise SystemExit("Live version description has an unexpected model-count shape") + live[0].update(description=description, snapshotLabel=f"Snapshot {snapshot_date}") + return out + + +def manifest_problems( + before: dict, after: dict, upgrade: driver.Upgrade | None = None +) -> list[tuple]: + """The manifest leaves that differ outside what the release may change. + + A leaf may differ only under a MANIFEST_CHANGES path, and never at a + MANIFEST_PINNED one: the pinned references and the reference explanations + keep release 20261006's hashes. + """ + pinned = MANIFEST_PINNED + changes = MANIFEST_CHANGES + if upgrade is not None: + pinned = pinned - UPGRADE_MANIFEST_UNPINNED + changes += UPGRADE_MANIFEST_CHANGES + return [ + path + for path in leaf_paths(before, after) + if path in pinned + or not any(path[: len(prefix)] == prefix for prefix in changes) + ] + + +def upgraded_reference_refresh(stage: Path, upgrade: driver.Upgrade) -> dict: + """The exact refresh leaves the retained sidecar writes to the manifest. + + The build gate checks the engine version; the freezer also needs the + complete bundle's provenance. Refuse missing or malformed fields before + any destination is written, with the same values used after freezing. + """ + meta = read_json(stage / driver.BUILD_COPY / driver.META_NAME) + regenerated = meta.get("regenerated_at_utc") + try: + if not isinstance(regenerated, str) or not re.match( + r"\d{4}-\d{2}-\d{2}T", regenerated + ): + raise ValueError + timestamp = datetime.datetime.fromisoformat(regenerated) + if timestamp.utcoffset() != datetime.timedelta(0): + raise ValueError + except ValueError: + raise SystemExit( + "The installed engine upgrade's regenerated_at_utc must be a " + "nonempty ISO timestamp in UTC" + ) from None + bundle = meta["policyengine_bundles"]["us"] + fields = { + "policyengine_version": "policyengine_version", + "policyengine_us_version": "model_version", + "policyengine_us_data_build_id": "certified_data_build_id", + "policyengine_us_dataset": "default_dataset", + "policyengine_us_dataset_uri": "default_dataset_uri", + "policyengine_us_data_artifact_sha256": "certified_data_artifact_sha256", + } + invalid = [ + field + for field in fields.values() + if not isinstance(bundle.get(field), str) or not bundle[field].strip() + ] + if invalid: + raise SystemExit( + "The installed engine upgrade's reference bundle metadata is " + f"incomplete or malformed: {invalid}" + ) + return { + "date": regenerated[:10], + "regenerated_at_utc": regenerated, + "reference_csv_sha256": upgrade.sha256[driver.CSV_NAME], + **{name: bundle[field] for name, field in fields.items()}, + } + + +def verify_frozen( + *, + stage: Path, + snapshot: Path, + annotations: Path, + exclusions_sha256: str, + pointer: dict, + window: str, + snapshot_date: str, + previous_serving: dict, + incumbents: set[str], + runs: dict[str, str], + upgrade: driver.Upgrade | None = None, +) -> None: + """What the freezer wrote is the release, checked file by file. + + The frozen references are release 20261006's and the frozen exclusion + record is the release's. The annotations directory holds the staged + adjudications and annotation CSVs, byte for byte, and the wording + amendments record; the frozen adjudication record is in the committed + form with the 2026-10-06 wave named. The serving configuration keeps + release 20261006's rows and adds the new model's. The manifest differs + from release 20261006's only where the release may change it, and states + the release's snapshot date, model count, response window, exclusion + counts, artifact and exclusion pin. + """ + staged_annotations = stage / "publish" / RUN / "annotations" + frozen_run = snapshot / "runs" / RUN + if upgrade is None: + driver.verify_reference_pins(frozen_run, "frozen reference", exclusions_sha256) + else: + for name, pin in driver.upgraded_reference_pins(upgrade).items(): + path = frozen_run / name + if not path.is_file() or digest(path) != pin: + raise SystemExit( + f"Frozen {name} is not the installed engine upgrade's file" + ) + path = snapshot / "us_reference_outputs.csv" + if not path.is_file() or digest(path) != upgrade.sha256[driver.CSV_NAME]: + raise SystemExit( + "Frozen us_reference_outputs.csv is not the installed engine " + "upgrade's file" + ) + copied = (ADJUDICATIONS, *ANNOTATION_CSVS, EXPLANATIONS) + frozen = sorted(path.name for path in annotations.iterdir() if path.is_file()) + if frozen != sorted((*copied, COMMITTED_AMENDMENTS)): + raise SystemExit(f"Frozen annotation files are not the release's: {frozen}") + for name in copied: + if (annotations / name).read_bytes() != ( + staged_annotations / name + ).read_bytes(): + raise SystemExit(f"The frozen {name} is not the staged one") + driver.verify_record_form( + (annotations / ADJUDICATIONS).read_text(), + driver.base_adjudication_record(), + driver.upgrade_wave(driver.upgrade_decisions(upgrade)), + ) + if (annotations / COMMITTED_AMENDMENTS).read_bytes() != committed_amendments(stage): + raise SystemExit( + f"The frozen {COMMITTED_AMENDMENTS} is not release 20261006's " + "record with the stage's list appended" + ) + serving = serving_problems( + read_json(snapshot / "model_serving_config.json"), + previous_serving, + incumbents, + runs, + ) + if serving: + raise SystemExit(f"Frozen serving configuration: {serving[:8]}") + manifest = read_json(snapshot / "manifest.json") + base = json.loads(driver.base_commit_blob(SNAPSHOT / "manifest.json")) + outside = manifest_problems(base, manifest, upgrade) + if outside: + named = [".".join(map(str, path)) for path in outside[:8]] + raise SystemExit(f"The frozen manifest changes outside the release: {named}") + artifact = {key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes")} + live = manifest["live_dashboard_artifact"] + excluded = manifest["reference_exclusions"] + stated = { + "snapshot_date": (manifest["snapshot_date"], snapshot_date), + "scope.models": (manifest["scope"]["models"], BOARD_MODELS), + "model_response_date": (manifest["model_response_date"], window), + "reference_exclusions.outputs": ( + excluded["outputs"], + driver.RELEASE_EXCLUSIONS if upgrade is None else upgrade.records, + ), + "reference_exclusions.scored_outputs_per_model": ( + excluded["scored_outputs_per_model"], + driver.RELEASE_SCORED if upgrade is None else upgrade.scored_outputs, + ), + "published_dashboard_artifact": ( + manifest["published_dashboard_artifact"], + artifact, + ), + "live_dashboard_artifact": ({key: live.get(key) for key in artifact}, artifact), + f"source_run_artifacts {EXCLUSIONS}": ( + manifest["source_run_artifacts"][RUN]["files"].get(EXCLUSIONS), + exclusions_sha256, + ), + } + if upgrade is not None: + stated.update( + { + f"source_run_artifacts {name}": ( + manifest["source_run_artifacts"][RUN]["files"].get(name), + pin, + ) + for name, pin in driver.upgraded_reference_pins(upgrade).items() + } + ) + stated["committed_snapshot_artifacts.us_reference_outputs.csv"] = ( + manifest["committed_snapshot_artifacts"].get("us_reference_outputs.csv"), + upgrade.sha256[driver.CSV_NAME], + ) + stated[f"audit_annotation_artifacts.files.{EXPLANATIONS}"] = ( + manifest["audit_annotation_artifacts"]["files"].get(EXPLANATIONS), + upgrade.sha256[EXPLANATIONS], + ) + refresh = upgraded_reference_refresh(stage, upgrade) + stated.update( + { + f"reference_output_refresh.{name}": ( + manifest["reference_output_refresh"].get(name), + value, + ) + for name, value in refresh.items() + } + ) + wrong = {name: got for name, (got, want) in stated.items() if got != want} + if wrong: + raise SystemExit(f"The frozen manifest does not state the release: {wrong}") + + +def configure_freezer( + freezer, + *, + snapshot_date: str, + source_run: Path, + payload_path: Path, + pointer: dict, + evidence_paths: dict[str, str], + cases_dir: Path, + previous_serving: dict, + incumbents: set[str], + runs: dict[str, str], + upgrade: driver.Upgrade | None = None, +) -> None: + """Point freeze_snapshot at the stage and this release. + + Read-only input paths are all in this checkout. They override defaults + that captured the old audit directory when freeze_snapshot was imported. + """ + freezer.SNAPSHOT_DATE = snapshot_date + freezer.SOURCE_RUN = source_run + freezer.SOURCE_US = source_run / "us" + freezer.SOURCE_ANNOTATIONS = source_run / "annotations" + freezer.REFERENCE_META_SOURCE = source_run / "us/reference_outputs.csv.meta.json" + freezer.REFERENCE_PINS = ( + None if upgrade is None else driver.upgraded_reference_pins(upgrade) + ) + freezer.PUBLISHED_DASHBOARD_SOURCE = payload_path + freezer.PUBLISHED_DASHBOARD_ARTIFACT = { + key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes") + } + freezer.RUN_STATE_EVIDENCE = evidence_paths + freezer.AUDIT_CASES_DIR = cases_dir + freezer.audit_judge_provenance = functools.partial( + freezer.audit_judge_provenance, cases_dir=cases_dir + ) + freezer.developer_adjudications_block = functools.partial( + freezer.developer_adjudications_block, cases_dir=cases_dir + ) + original_freeze_serving = freezer.freeze_serving_configuration + + def freeze_serving(destination: Path) -> None: + original_freeze_serving(destination) + serving = merge_serving( + read_json(destination), previous_serving, incumbents, runs + ) + serving["registry_commit"] = freezer._serving_registry_commit(serving) + write_json(destination, serving) + + freezer.freeze_serving_configuration = freeze_serving + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--stage-dir", type=Path, required=True) + parser.add_argument("--tag", default=driver.RELEASE_TAG) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + if not re.fullmatch(r"dashboard-data-\d{8}[a-z]?", args.tag): + parser.error("--tag must be dashboard-data-YYYYMMDD[a-z]") + stage = args.stage_dir.resolve() + if not stage.is_relative_to(ROOT): + parser.error("--stage-dir must be inside this checkout") + import freeze_snapshot as freezer + import pandas as pd + + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + pd.options.future.infer_string = False + + from policybench.dashboard_schema import validate_dashboard_payload + + verify_freezer_destinations(freezer) + payload_path = stage / f"data-board{BOARD_MODELS}.json" + receipt = verify_receipt(stage, payload_path, args.tag) + spec = release_spec(args.tag) + upgrade = verify_upgrade(stage, receipt) + snapshot_date = spec["snapshot_date"] + # The release's exclusion record, rebuilt from git and the spec: what the + # stage must score on and the freeze must commit. + if upgrade is None: + exclusions = driver.build_release_exclusions( + driver.base_exclusion_record(), spec + ) + exclusions_text = driver.exclusions_text(exclusions) + else: + exclusions_text = upgrade.exclusions_text + exclusions = json.loads(exclusions_text) + exclusions_sha256 = verify_installed_exclusions( + stage, receipt, exclusions_text, upgrade + ) + # Claude Haiku 5.5's run files are the committed pins' and its bundle rows + # are its run file's, every column: its cost, tokens and latency included. + driver.verify_new_model_inputs(stage) + runs = evidence_runs() + payload_hash = digest(payload_path) + payload = read_json(payload_path) + recombined = json.dumps({"countries": {"us": payload["countries"]["us"]}}) + if recombined.encode() != payload_path.read_bytes(): + raise SystemExit("Payload does not recombine to the freeze format") + errors = validate_dashboard_payload(payload, require_failure_annotations=True) + if errors: + raise SystemExit(f"Strict dashboard gate failed: {errors[:5]}") + stats = payload["countries"]["us"]["modelStats"] + model_names = {row["model"] for row in stats} + if len(stats) != BOARD_MODELS or any( + row["condition"] != "no_tools" for row in stats + ): + raise SystemExit( + f"Release must contain exactly {BOARD_MODELS} no-tools model rows" + ) + + snapshot = ROOT / SNAPSHOT + frozen_run = snapshot / "runs" / RUN + # The incumbents' frozen serving evidence is release 20261006's, from git. + previous_serving = base_serving_config() + incumbents = set(previous_serving["models"]) - set(NEW_MODELS.values()) + if len(incumbents) != driver.BASE_MODELS or model_names != incumbents | set( + NEW_MODELS.values() + ): + raise SystemExit( + f"Frozen incumbent roster and staged {BOARD_MODELS}-model roster disagree" + ) + verify_scored_outputs(payload["countries"]["us"], exclusions["exclusions"], upgrade) + source_run = stage / "publish" / RUN + source_us = source_run / "us" + staged_annotations = source_run / "annotations" + verify_references( + source_us, + frozen_run, + read_json(snapshot / "manifest.json"), + exclusions_sha256, + upgrade, + ) + verify_explanations(staged_annotations, upgrade, stage=stage) + if upgrade is not None: + # The engine-setup note must be supported before any destination is + # written, including the pointer. Restore the freezer's input path so + # dry-run leaves its configuration alone. + previous_meta = freezer.REFERENCE_META_SOURCE + try: + freezer.REFERENCE_META_SOURCE = source_us / driver.META_NAME + setup = freezer.read_reference_engine_setup() + if setup["convention_count"] not in freezer.NUMBER_WORDS: + raise SystemExit( + "The installed engine upgrade names an unsupported " + f"publication convention count: {setup['convention_count']}" + ) + finally: + freezer.REFERENCE_META_SOURCE = previous_meta + upgraded_reference_refresh(stage, upgrade) + freezer.reference_policyengine_bundles( + source_us / driver.CSV_NAME, + "us", + require_digest=True, + manifest_reference_sha256=upgrade.sha256[driver.CSV_NAME], + ) + del payload, stats + # Export's own build, the incumbents' scope check included. + base = driver.base_payload_from_commit() + rebuild_payload(stage, receipt, payload_path, base, upgrade) + del base + verify_verdicts(stage) + rejudged = driver.rejudged_cases(stage) + verify_judge_provenance(stage, receipt, rejudged) + amendments = driver.load_amendments(stage, rejudged) + cases_dir = stage / "audit" / "cases" + added = verify_adjudication_record( + staged_annotations / ADJUDICATIONS, + source_us, + rejudged, + amendments, + cases_dir, + upgrade, + ) + verify_annotation_amendments(staged_annotations, amendments, stage / "audit") + previous_rows = base_prediction_rows() + staged_rows = pd.read_csv(source_us / "predictions.csv", low_memory=False) + for model in sorted(incumbents): + if freezer._model_prediction_rows_sha256(previous_rows, model) != ( + freezer._model_prediction_rows_sha256(staged_rows, model) + ): + raise SystemExit(f"Incumbent prediction evidence changed: {model}") + del previous_rows + + evidence_paths = {} + for slug, model in NEW_MODELS.items(): + state_path = stage / "inputs" / slug / "run_state.json" + state = read_json(state_path) + if ( + state.get("model") != model + or state.get("completed") != 100 + or state.get("total") != 100 + or state.get("stopped_reason") is not None + or not isinstance(state.get("treatment_fingerprint"), dict) + ): + raise SystemExit( + f"Missing completed run or treatment evidence: {state_path}" + ) + validate_treatment(freezer, state, state_path, source_us / "scenarios.csv") + evidence = freezer._run_state_prediction_evidence( + state_path, staged_rows, model + ) + if evidence["kind"] != "run_state": + raise SystemExit( + f"Run-state prediction evidence failed for {model}: {evidence}" + ) + evidence_paths[model] = str(state_path) + del staged_rows + # The freezer dates the response window from these rows: from the run + # label's start to the last answer, not to the release date. Refuse here, + # before the pointer is written, rather than partway through the freeze. + window = freezer.model_response_window( + source_us / "predictions.csv", + freezer.MODEL_RESPONSE_START, + freezer.MODEL_RESPONSE_DATE, + ) + + adjudications = freezer.load_adjudications(staged_annotations / ADJUDICATIONS) + from policybench.adjudications import verify_adjudications_applied + + verify_adjudications_applied( + pd.read_csv(staged_annotations / "us_audit_row_annotations.csv"), + pd.read_csv(staged_annotations / "us_case_notes.csv"), + adjudications, + ) + freezer.verify_adjudications_keep_judge_verdicts(adjudications, cases_dir) + pointer = release_pointer(args.tag, payload_path, payload_hash) + versions_path = ROOT / VERSIONS + versions = release_versions( + read_json(versions_path), exclusions["exclusions"], snapshot_date, upgrade + ) + if args.dry_run: + references = ( + "references unchanged" + if upgrade is None + else f"references upgraded to {upgrade.engine_version}" + ) + records = driver.RELEASE_EXCLUSIONS if upgrade is None else upgrade.records + new = ( + driver.NEW_EXCLUSIONS + if upgrade is None + else driver.NEW_EXCLUSIONS + - len(upgrade.regenerated_ruled) + + len(upgrade.added) + ) + print( + f"Validated local release inputs: {args.tag}, {BOARD_MODELS} models, " + f"{payload_hash}; {references}; {records} " + f"exclusions ({new} new); {added} adjudications " + f"added beyond the ruled outputs; {len(amendments)} wording " + f"amendments; model responses {window}; snapshot {snapshot_date}; " + "nothing written" + ) + return + + configure_freezer( + freezer, + snapshot_date=snapshot_date, + source_run=source_run, + payload_path=payload_path, + pointer=pointer, + evidence_paths=evidence_paths, + cases_dir=cases_dir, + previous_serving=previous_serving, + incumbents=incumbents, + runs=runs, + upgrade=upgrade, + ) + write_json(ROOT / POINTER, pointer) + write_json(versions_path, versions) + freezer.ANNOTATIONS_DEST.mkdir(parents=True, exist_ok=True) + shutil.copyfile( + staged_annotations / ADJUDICATIONS, + freezer.ANNOTATIONS_DEST / ADJUDICATIONS, + ) + freezer.main() + freeze_amendments(stage, freezer.ANNOTATIONS_DEST) + # The freezer copies the staged references, exclusions and annotations + # byte for byte and builds the manifest and the serving rows; confirm it. + verify_frozen( + stage=stage, + snapshot=snapshot, + annotations=freezer.ANNOTATIONS_DEST, + exclusions_sha256=exclusions_sha256, + pointer=pointer, + window=window, + snapshot_date=snapshot_date, + previous_serving=previous_serving, + incumbents=incumbents, + runs=runs, + upgrade=upgrade, + ) + cache = ROOT / "app/.cache" / f"dashboard-data-{payload_hash[:16]}.json" + cache.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(payload_path, cache) + asset = stage / "predictions.csv.gz" + freezer.gzip_deterministic( + source_us / "predictions.csv", asset, stored_name="predictions.csv" + ) + if asset.read_bytes() != (frozen_run / "predictions.csv.gz").read_bytes(): + raise SystemExit("The predictions asset is not the frozen predictions") + print(f"Built {args.tag} locally; payload and predictions remain in {stage}") + print("Next: update note and paper prose, render the paper, re-pin, run tests.") + + +if __name__ == "__main__": + main() diff --git a/scripts/recover_20261010_audit.py b/scripts/recover_20261010_audit.py new file mode 100644 index 00000000..3921e494 --- /dev/null +++ b/scripts/recover_20261010_audit.py @@ -0,0 +1,138 @@ +"""Rebuild release dashboard-data-20261010's judged audit tree (677 cases). + +The release's stage (results/local/final/pe2.38.6/stage in the removed +release-haiku55 worktree) was deleted on 2026-10-10, with the 232 verdicts its +Opus 5.5 judges wrote. Each judge was a Claude Code session whose transcript +is still in the judge login's CLAUDE_CONFIG_DIR (projects/*/.jsonl). +The runner (scripts/run_audit_claude.sh) wrote + prompt.md = the bytes the judge read (its one user text message) + verdict.json = json.dumps(accepted StructuredOutput input, indent=2, sort_keys=True) +so both are recoverable from the transcript, and each must hash to the sha256 +docs/haiku55/judge_provenance.json committed for its case. + +The tree is release 20261006's seed (674 judged cases, which must match +docs/haiku55/seed_digest.csv) with the 232 recovered cases written over it. +Usage: recover_20261010_audit.py +Read-only on its inputs; refuses to write into an existing out dir. +""" +import csv, glob, hashlib, json, os, shutil, sys +from pathlib import Path + +repo, seed_dir, out = (Path(a) for a in sys.argv[1:4]) +CONFIGS = { + "pb-judge": Path.home() / ".policybench-judge/pb-judge-config", + "claude-18": Path.home() / ".policybench-judge/claude-18-config", +} +NUDGE = "[structured-output-enforce] You MUST call the StructuredOutput tool to complete this request. Call this tool now." +sha = lambda b: hashlib.sha256(b).hexdigest() + +def fail(msg): + sys.exit(f"REFUSED: {msg}") + +# 1. The 20261006 seed tree must be the committed digest. +digest = {r["case_id"]: r for r in csv.DictReader(open(repo / "docs/haiku55/seed_digest.csv"))} +cases = sorted(p for p in (seed_dir / "cases").iterdir() if (p / "prompt.md").is_file()) +got = {c.name: {"prompt_sha256": sha((c / "prompt.md").read_bytes()), + "verdict_sha256": sha((c / "verdict.json").read_bytes())} for c in cases} +if set(got) != set(digest) or any(got[k]["prompt_sha256"] != digest[k]["prompt_sha256"] + or got[k]["verdict_sha256"] != digest[k]["verdict_sha256"] for k in got): + fail("the 20261006 seed tree is not docs/haiku55/seed_digest.csv") +print(f"20261006 seed: {len(got)} judged cases match the committed digest") + +# 2. The 232 committed provenance entries. +prov = json.load(open(repo / "docs/haiku55/judge_provenance.json"))["verdicts"] +want = {e["case_id"]: e for e in prov} +by_prompt = {} +for e in prov: + by_prompt.setdefault(e["prompt_sha256"], []).append(e["case_id"]) +print(f"provenance: {len(want)} re-judged cases") + +# 3. Scan every judge transcript; keep the ones whose prompt and verdict hash match. +found = {} +scanned = 0 +for login, cfg in CONFIGS.items(): + for path in sorted(glob.glob(str(cfg / "projects/*/*.jsonl"))): + scanned += 1 + texts, structured, refused, version, session = [], [], set(), set(), None + for line in open(path, encoding="utf-8"): + ev = json.loads(line) + session = ev.get("sessionId", session) + if ev.get("version"): + version.add(ev["version"]) + content = (ev.get("message") or {}).get("content") + if ev.get("type") == "user" and isinstance(content, str): + if not (ev.get("isMeta") is True and content == NUDGE): + texts.append(content) + if isinstance(content, list): + structured += [p for p in content if isinstance(p, dict) + and p.get("type") == "tool_use" and p.get("name") == "StructuredOutput"] + refused |= {p.get("tool_use_id") for p in content if isinstance(p, dict) + and p.get("type") == "tool_result" and p.get("is_error")} + if len(texts) != 1: + continue + prompt = texts[0].encode("utf-8") + candidates = by_prompt.get(sha(prompt), []) + accepted = [p for p in structured if p.get("id") not in refused] + if not candidates or len(accepted) != 1: + continue + verdict = json.dumps(accepted[0]["input"], indent=2, sort_keys=True).encode("utf-8") + for case in candidates: + if sha(verdict) == want[case]["verdict_sha256"]: + if case in found and found[case]["verdict"] != verdict: + fail(f"two transcripts give {case} different verdict bytes") + found.setdefault(case, {"prompt": prompt, "verdict": verdict, "transcript": path, + "login": login, "session_id": session, + "cli_versions": sorted(version)}) +print(f"scanned {scanned} transcripts; recovered {len(found)} of {len(want)}") +missing = sorted(set(want) - set(found)) +if missing: + fail(f"{len(missing)} cases not recoverable: {missing[:10]}") + +# 4. Write the tree: the 20261006 seed, then the recovered cases over it. +if out.exists(): + fail(f"{out} exists") +shutil.copytree(seed_dir, out, symlinks=False) +record = [] +for case, item in sorted(found.items()): + d = out / "cases" / case + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + (d / "prompt.md").write_bytes(item["prompt"]) + (d / "verdict.json").write_bytes(item["verdict"]) + shutil.copyfile(item["transcript"], d / "claude.transcript.jsonl") + e = want[case] + meta = { + "judge_runner": "scripts/run_audit_claude.sh", + "verdict_sha256": e["verdict_sha256"], + "prompt_sha256": e["prompt_sha256"], + "judge_template_version": 1, + "judge_model_reported": e["judge_model_reported"], + "judge_effort": e["judge_effort"], + "judge_account_declared": e["judge_account_declared"], + "judged_at_utc": e["judged_at_utc"], + "session_id": item["session_id"], + "judge_cli_version": item["cli_versions"][0] if len(item["cli_versions"]) == 1 else item["cli_versions"], + "recovered": { + "why": "release 20261010's stage was deleted with its worktree on 2026-10-10; " + "the original sidecar was lost with it", + "from": f"the judge's Claude Code session transcript ({item['login']} login)", + "checks": "prompt.md and verdict.json hash to docs/haiku55/judge_provenance.json", + "on": "2026-10-12", + }, + } + (d / "verdict.meta.json").write_text(json.dumps(meta, indent=2, sort_keys=True)) + record.append({"case_id": case, "login": item["login"], "session_id": item["session_id"], + "prompt_sha256": e["prompt_sha256"], "verdict_sha256": e["verdict_sha256"]}) + +# 5. The rebuilt tree's digest is the 20261006 digest overlaid with the provenance. +final = {c.name: (sha((c / "prompt.md").read_bytes()), sha((c / "verdict.json").read_bytes())) + for c in sorted((out / "cases").iterdir()) if (c / "prompt.md").is_file()} +expect = {k: (v["prompt_sha256"], v["verdict_sha256"]) for k, v in digest.items()} +expect.update({k: (v["prompt_sha256"], v["verdict_sha256"]) for k, v in want.items()}) +if final != expect: + fail("rebuilt tree does not match the 20261006 digest overlaid with the provenance") +text = "case_id,prompt_sha256,verdict_sha256\n" + "".join(f"{k},{p},{v}\n" for k, (p, v) in sorted(final.items())) +(out.parent / "seed_digest_20261010.csv").write_text(text) +json.dump({"recovered": record}, open(out.parent / "recovery_record.json", "w"), indent=1) +print(f"rebuilt {len(final)} judged cases; digest sha256 {sha(text.encode())}") From 23e1dd715d8ba2d2b24a56a0dbfe4b32cb1d1f18 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 19:09:33 -0400 Subject: [PATCH 03/28] Add the CA educator-expense addback fix module (r33) for scenario_099's regeneration target Co-Authored-By: Claude Opus 5.5 --- .../fixes/r33_ca_educator_expense_addback.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 reference_audit/2026-10-12-engine-upgrade/fixes/r33_ca_educator_expense_addback.py diff --git a/reference_audit/2026-10-12-engine-upgrade/fixes/r33_ca_educator_expense_addback.py b/reference_audit/2026-10-12-engine-upgrade/fixes/r33_ca_educator_expense_addback.py new file mode 100644 index 00000000..411c5027 --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/fixes/r33_ca_educator_expense_addback.py @@ -0,0 +1,72 @@ +"""r33: California AGI adds back the federal educator-expense deduction. + +Defect (policyengine-us 2.38.6 and earlier) +-------------------------------------------- +variables/gov/states/ca/tax/income/ca_additions.py adds only ca_hsa_addition to +federal AGI, so California AGI keeps the federal above-the-line educator +expense deduction (educator_expense_ald, IRC 62(a)(2)(D)). + +California law +-------------- +California does not conform to the federal educator-expense deduction: +R&TC 17072(b), and the FTB's 2025 Instructions for Schedule CA (540), +Section C, line 11 ("California law does not conform to federal law regarding +educator expenses"), which restores the federally claimed amount to +California AGI. Release 20261010's Opus 5.5 judge flagged it on CA +scenario_099's state income tax (reference_audit/2026-10-09-engine-upgrade/ +final_actions.json, excluded_rechecked). + +Fix +--- +ca_educator_expense_addition adds educator_expense_ald for California tax +units, and ca_additions adds it to ca_hsa_addition: the two variables +PolicyEngine/policyengine-us#10052 (merged 2026-10-12, released in 2.41.0) +defines, with the same adds lists. On an engine that already has them the +reform replaces each with an identical definition and changes nothing. +""" + +from __future__ import annotations + +from policyengine_core.reforms import Reform +from policyengine_us.model_api import * # noqa: F401,F403 + +FIX_ID = "r33_ca_educator_expense_addback" +DESCRIPTION = ( + "California AGI adds back the federal educator-expense deduction " + "(R&TC 17072(b); Schedule CA (540) Section C line 11)." +) + + +class ca_educator_expense_addition(Variable): + value_type = float + entity = TaxUnit + label = "California educator expense addition" + documentation = ( + "Federal above-the-line educator expense deduction claimed on this " + "return, restored to California AGI under R&TC 17072(b)." + ) + unit = USD + definition_period = YEAR + defined_for = StateCode.CA + reference = ( + "https://leginfo.legislature.ca.gov/faces/codes_displaySection.xhtml?lawCode=RTC§ionNum=17072", + "https://www.ftb.ca.gov/forms/2025/2025-540-booklet.pdf#page=64", + ) + adds = ["educator_expense_ald"] + + +class ca_additions(Variable): + value_type = float + entity = TaxUnit + label = "CA AGI additions to federal AGI" + unit = USD + definition_period = YEAR + reference = "https://www.ftb.ca.gov/forms/2021/2021-540.pdf" + defined_for = StateCode.CA + adds = ["ca_hsa_addition", "ca_educator_expense_addition"] + + +class reform(Reform): + def apply(self): + self.update_variable(ca_educator_expense_addition) + self.update_variable(ca_additions) From ac05eefbf4a8e1549023eb3f27403aa3899f444e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 19:16:50 -0400 Subject: [PATCH 04/28] WIP: second-upgrade builder, evidence on 2.38.6 and mixed-template install reference_audit/2026-10-12-engine-upgrade/ adapts release 20261010's builder, sweep timing, narratives and action drafting to base 5a8164a0 and policyengine-us 2.38.6. Its evidence on 2.38.6 gives the audited targets of the ten cells the newest engine is expected to fix: Idaho 007/053 (#10023), Wisconsin 042/091 (#10026), SNAP 080 (#10033), EITC 100 and 119 (#10030), and CA 099 state with the new r33 educator-expense addback (#10052). A target may name a fix module this audit adds, at its own path (fix_location), and the freeze's target-pin check accepts such a path. The driver renders install-references' audit per case on the template its sidecar records, so the seed's v1 verdicts survive and re-opened cases take v2. New verdicts must be on v2, and the public provenance record states each verdict's template. The freeze counts exclusions on three earlier engines. Co-Authored-By: Claude Opus 5.5 --- docs/musewonder/spec.json | 17 + .../actions_from_cells.py | 591 ++++++ .../evidence/pe2.38.6.json | 116 ++ .../scripts/build_references_upgrade.py | 1715 +++++++++++++++++ .../scripts/check_against_evidence.py | 111 ++ .../scripts/narratives_upgrade.py | 429 +++++ .../scripts/sweep_timing.py | 790 ++++++++ scripts/date_musewonder_judge_verdicts.py | 112 +- scripts/finish_musewonder.py | 176 +- scripts/freeze_musewonder.py | 184 +- scripts/freeze_snapshot.py | 16 +- 11 files changed, 4066 insertions(+), 191 deletions(-) create mode 100644 docs/musewonder/spec.json create mode 100644 reference_audit/2026-10-12-engine-upgrade/actions_from_cells.py create mode 100644 reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json create mode 100644 reference_audit/2026-10-12-engine-upgrade/scripts/build_references_upgrade.py create mode 100644 reference_audit/2026-10-12-engine-upgrade/scripts/check_against_evidence.py create mode 100644 reference_audit/2026-10-12-engine-upgrade/scripts/narratives_upgrade.py create mode 100644 reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py diff --git a/docs/musewonder/spec.json b/docs/musewonder/spec.json new file mode 100644 index 00000000..68bbc88a --- /dev/null +++ b/docs/musewonder/spec.json @@ -0,0 +1,17 @@ +{ + "note": "Inputs of the release after dashboard-data-20261010, read by scripts/finish_musewonder.py. The release folds Meta Muse Wonder onto release dashboard-data-20261010 and moves the references from policyengine-us 2.38.6 to the newest release, regenerating the excluded outputs whose engine defects it fixes (Max 2026-09-28, 2026-10-09 and 2026-10-12). It rules no exclusion: d1252 (VA scenario_039 federal and OH scenario_025 state income tax, held references) and d1263 (the Tax Table against the rate schedule) were open when it was staged, so nothing scored changes on their points.", + "release_tag": "dashboard-data-20261013", + "base_tag": "dashboard-data-20261010", + "base_sha256": "f834478e65191f8426ef479506f9f6fe32c2e7107f30fdbf4e7ba0e6acba6410", + "rulings": { + "engine": "Max, 2026-09-28: 'we should be using the latest pe for this always!'; 2026-10-09: 'yes i want to wait for hte fixed engine'; 2026-10-12, of the fixes release 20261010 left excluded: 'If any have shipped, regenerate their cells instead of excluding them.'", + "d1252": "open when staged: no change to VA scenario_039 federal or OH scenario_025 state income tax", + "d1263": "open when staged: federal references keep the section 1 rate schedule" + }, + "proposals": [], + "adjudications": [], + "restated_adjudications": [], + "upgrade_adjudications": [], + "triage_adjudications": [], + "snapshot_date": "2026-10-13" +} diff --git a/reference_audit/2026-10-12-engine-upgrade/actions_from_cells.py b/reference_audit/2026-10-12-engine-upgrade/actions_from_cells.py new file mode 100644 index 00000000..570209a4 --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/actions_from_cells.py @@ -0,0 +1,591 @@ +"""Draft an actions file for build_references_upgrade.py from a first compute pass. + +Adapted from reference_audit/2026-10-09-engine-upgrade/actions_from_cells.py. +Input: the builder's first-pass computed.csv on the new release (every output +against the 20261010 references: fix, scenario_id, state, variable, frozen, +recomputed, delta, moved), or a cells table in the US hub's cells_pe.csv +form, the release spec (docs/musewonder/spec.json, through +scripts/finish_musewonder.py) and the exclusion records (20261010's, read from +git; the release rules none). + +Output: a DRAFT actions file ("draft": true, which the builder refuses until a +reviewer clears it) and, on stdout, every judgment call it drafted. Nothing is +decided silently: each moved output is classified by + + status scored, excluded in 20261010, or ruled in the release; + audited target whether the new value lands within $1 of its record's + corrected value (alternative_value) or, with --evidence, of + the corrected value the record's fix modules give on a + pre-fix engine (a "fix_modules" target, which the builder + also checks the modules leave unchanged on the new engine); + and the hub's own column; + reason engine defect, unlisted input, or a ruled reason (later law, + household scope); + +and lands in exactly one place: + + regenerated_exclusions an engine-defect record the new engine lands on + (PROPOSED: confirm the upstream fix and its release); + excluded_rechecked any other excluded output that moves, with a drafted + reason (PROPOSED: check the reason); + approved / new_exclusions a scored move a RULES entry recognizes, with drafted + cause and basis or a drafted record (PROPOSED: check every + sentence against the build's trace); + review only a scored move no rule recognizes (UNDECIDED: the builder + refuses it as an unreviewed move until a reviewer adds it). + +kept_exclusions_from_release lists every ruled output that is not regenerated. + + PYTHONPATH= python actions_from_cells.py \\ + --computed /computed.csv --engine X.Y.Z [--date YYYY-MM-DD] \\ + [--evidence reference_audit/2026-10-12-engine-upgrade/evidence/pe.json] \\ + [--upstreams reference_audit/2026-10-12-engine-upgrade/upstreams.json] \\ + --out actions.draft.json +""" + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE / "scripts")) + +from build_references_upgrade import ( # noqa: E402 + ENGINE_DEFECT, + PREVIOUS_ENGINE, + ROOT, + TARGET_FIX_MODULES, + UNNAMED_UPSTREAM, + Refusal, + beyond, + key_of, + load_base, + load_evidence, + release_record, + sha256, +) + +UNLISTED = "reference_depends_on_unlisted_input" +CELL_TOL = 0.006 # the hub's table rounds to cents +TARGET_TOL = 1.0 + + +@dataclass(frozen=True) +class Cell: + scenario_id: str + state: str + variable: str + previous: float + new: float + status: str + audited_target: str + + @property + def key(self) -> tuple[str, str]: + return (self.scenario_id, self.variable) + + +def money(value: float) -> str: + return f"${value:,.2f}" + + +# --- Rules for scored moves ------------------------------------------------- +# Each rule recognizes one documented kind of scored move and drafts its action. +# The text states what the hub documented (README.md, HUB_BRIEF.md) and what the +# 2.36.0 rehearsal's traces show (local_income_tax = in_county_tax with county_str +# ADAMS_COUNTY_IN for scenario_015 and 067; state_income_tax_before_refundable_credits +# = id_income_tax_before_refundable_credits + id_pbf $10 for scenario_076); a +# reviewer checks it against the build's trace. + + +def _indiana_county(cell: Cell, engine: str, date: str) -> dict: + return { + "action": "new_exclusions", + "rule": "indiana_county_unlisted", + "entry": { + "scenario_id": cell.scenario_id, + "variable": cell.variable, + "reason_code": UNLISTED, + "unlisted_input": "county of residence (Indiana county income tax)", + "alternative_reading": ( + "The prompt places the household in Indiana but states no county, " + "and Indiana's county income taxes (IC 6-3.6) differ by county. " + f"{engine} computes Indiana county tax (in_county_tax) in " + "local_income_tax at the rate of the county it assigns a household " + f"with no stated county (Adams County on {engine}). " + f"Read without a county, as {PREVIOUS_ENGINE} did, the household " + "owes no county tax." + ), + "frozen_value": cell.new, + "alternative_value": cell.previous, + "engine_version": engine, + "decided_on": date, + "decided_by": "developer", + "note": ( + f"Found in the {date} engine upgrade (the US hub's sweep, " + "cells_pe.md: local-tax standardization wires Indiana county " + "income tax into local_income_tax). The alternative is the " + f"{PREVIOUS_ENGINE} reference." + ), + }, + "call": ( + "NEW EXCLUSION (unlisted input: county) - confirm the county the engine " + "assumes in the build trace and the alternative value" + ), + } + + +def _idaho_building_fund(cell: Cell, engine: str, date: str) -> dict: + return { + "action": "approved", + "rule": "idaho_permanent_building_fund", + "entry": { + "scenario_id": cell.scenario_id, + "variable": cell.variable, + "value": cell.new, + "cause": "id_permanent_building_fund_tax", + "basis": ( + "Idaho's $10 permanent building fund tax counts in state income " + f"tax: on {engine}, state_income_tax_before_refundable_credits adds " + "id_pbf ($10 for a filer liable for it, id_pbf_liable; Idaho Form 40 " + "instructions) to Idaho income tax before " + "refundable credits, since policyengine-us 2.30.0 " + "(PolicyEngine/policyengine-us#9810)." + ), + }, + "call": ( + "APPROVE (+$10 permanent building fund tax, id_pbf in the 2.36.0 " + "trace) - confirm the trace and the Idaho Code citation" + ), + } + + +RULES = ( + ( + lambda c: ( + c.state == "IN" + and c.variable == "local_income_tax" + and c.previous == 0 + and c.new > 0 + ), + _indiana_county, + ), + ( + lambda c: ( + c.state == "ID" + and c.variable == "state_income_tax_before_refundable_credits" + and abs(c.new - c.previous - 10) <= CELL_TOL + ), + _idaho_building_fund, + ), +) + + +# --- Inputs -------------------------------------------------------------------- + + +def read_cells(path: Path, board: dict, computed: dict | None) -> list[Cell]: + with Path(path).open(newline="") as source: + rows = list(csv.DictReader(source)) + keys = {(r["scenario_id"], r["variable"]) for r in rows} + if keys != set(board) or len(rows) != len(board): + raise Refusal("the cells table does not list exactly the 20261010 outputs") + cells = [] + for r in rows: + k = (r["scenario_id"], r["variable"]) + if abs(float(r["frozen_20261010"]) - board[k]) > CELL_TOL: + raise Refusal(f"{k}: the table's frozen value is not 20261010's") + new = float(r["new"]) + if computed is not None: + if abs(computed[k] - new) > CELL_TOL: + raise Refusal(f"{k}: computed {computed[k]} is not the table's {new}") + new = computed[k] + cells.append( + Cell( + k[0], + r["state"], + k[1], + board[k], + new, + r["status"], + r.get("audited_target", ""), + ) + ) + return cells + + +def cells_from_computed(path: Path, board: dict, records: dict, base_excluded: set) -> list[Cell]: + """Every output of a builder pass's computed.csv as a cell: its 20261010 + reference (which must be the row's frozen column), its new value and its + status. No hub column: the audited target is left to the evidence.""" + with Path(path).open(newline="") as source: + rows = list(csv.DictReader(source)) + keys = [(r["scenario_id"], r["variable"]) for r in rows] + if set(keys) != set(board) or len(keys) != len(board): + raise Refusal("computed.csv does not list exactly the 20261010 outputs") + cells = [] + for r, k in zip(rows, keys): + if abs(float(r["frozen"]) - board[k]) > 1e-6: + raise Refusal(f"{k}: computed.csv's frozen value is not 20261010's") + status = ( + "excluded 20261010: " + str(records[k].get("root_cause") or records[k]["reason_code"]) + if k in base_excluded + else ("ruled" if k in records else "scored") + ) + cells.append(Cell(k[0], r["state"], k[1], board[k], float(r["recomputed"]), status, "")) + return cells + + +def read_computed(path: Path) -> dict: + with Path(path).open(newline="") as source: + return { + (r["scenario_id"], r["variable"]): float(r["recomputed"]) + for r in csv.DictReader(source) + } + + +def engine_from_path(path: Path) -> str: + match = re.fullmatch(r"cells_pe(\d+\.\d+\.\d+)\.csv", Path(path).name) + if not match: + raise Refusal(f"cannot read the engine from {Path(path).name}; pass --engine") + return match.group(1) + + +# --- Drafting ------------------------------------------------------------------ + + +def first_upgrade_rechecks(meta: dict) -> dict: + """The base's last engine upgrade's rechecks (release 20261010's, on 2.38.6).""" + last = [r for r in meta["revisions"] if r["kind"] == "engine_upgrade"][-1] + return {key_of(r): r for r in last["excluded_outputs_rechecked"]} + + +def second_reason(record: dict) -> str: + """The record's note when it names an unlisted input besides the defect.""" + note = str(record.get("note", "")) + return note if "unlisted input" in note.lower() else "" + + +def record_basis(record: dict) -> str: + return str( + record.get("unlisted_input") + or record.get("root_cause") + or record["reason_code"] + ) + + +def upstream_for(record: dict, key, upstreams: dict | None) -> str: + """The upstream fix a regeneration names: the output's own entry in + ``upstreams``, else each of its root causes' (joined with "; "), else the + record's own upstream.""" + if not upstreams: + return record.get("upstream", "") + own = upstreams.get("outputs", {}).get(f"{key[0]}|{key[1]}") + if own: + return own + causes = str(record.get("root_cause", "")).split("+") + fixes = [upstreams.get("root_causes", {}).get(cause) for cause in causes] + if causes and all(fixes): + return "; ".join(dict.fromkeys(fixes)) + return record.get("upstream", "") + + +def draft_actions( + cells: list[Cell], + release: dict, + ruled: set, + base_excluded: set, + first_rechecks: dict, + *, + engine_version: str, + date: str, + spec_sha256: str, + precise: bool, + evidence: dict | None = None, + upstreams: dict | None = None, +) -> tuple[dict, list[dict]]: + """The draft actions file and the judgment calls, one per moved output. + + ``evidence`` is {"ref": {path, sha256}, "doc": the loaded evidence file}: + a pre-fix engine's values for engine-defect outputs without and with + their fix modules.""" + evidence_items = {} + if evidence is not None: + for item in evidence["doc"]["items"]: + evidence_items[key_of(item)] = item + engine = f"policyengine-us {engine_version}" + ver_key = "value_on_2_38_6" + records = {key_of(r): r for r in release["exclusions"]} + actions = { + "engine": engine, + "previous_engine": PREVIOUS_ENGINE, + "date": date, + "release_spec_sha256": spec_sha256, + "draft": True, + "approved": [], + "regenerated_exclusions": [], + "new_exclusions": [], + "excluded_rechecked": [], + "kept_exclusions_from_release": [], + } + calls = [] + regenerated = set() + + def call(cell: Cell, decision: str, text: str, where: str) -> None: + calls.append( + { + "scenario_id": cell.scenario_id, + "state": cell.state, + "variable": cell.variable, + "previous": cell.previous, + "new": cell.new, + "status": cell.status, + "hub_audited_target": cell.audited_target, + "decision": decision, + "placed_in": where, + "call": text, + } + ) + + for cell in cells: + k = cell.key + if not beyond(cell.variable, cell.previous, cell.new): + continue + if k in records: + record = records[k] + source = "20261010" if k in base_excluded else "the release's ruling" + kept_value = cell.previous + if record["reason_code"] == ENGINE_DEFECT: + + def lands(aim: float) -> bool: + return abs(cell.new - aim) <= TARGET_TOL and not beyond( + cell.variable, aim, cell.new + ) + + target = float(record["alternative_value"]) + target_spec = {"kind": "record"} + target_text = f"the corrected value {money(target)} the " + target_text += f"{record['decided_on']} record computed" + item = evidence_items.get(k) + if not lands(target) and item is not None: + target = float(item["corrected_value"]) + target_spec = { + "kind": TARGET_FIX_MODULES, + "modules": list(item["modules"]), + "evidence": dict(evidence["ref"]), + } + target_text = ( + f"the corrected value {money(target)} its fix modules " + f"({' + '.join(item['modules'])}) give on " + f"{evidence['doc']['engine']}, which still has the defect " + f"({money(float(item['engine_value']))} without them)" + ) + on_target = lands(target) + second = second_reason(record) + if on_target and second: + reason = ( + f"Stays excluded on {engine}: the engine gives " + f"{money(cell.new)}, on the record's corrected value " + f"{money(target)}, but the record names a second reason " + f"that a new engine does not settle ({second}); the record " + f"keeps {money(kept_value)}." + ) + actions["excluded_rechecked"].append( + {"scenario_id": k[0], "variable": k[1], "reason": reason} + ) + call( + cell, + "PROPOSED", + f"RECHECK ({source} engine defect {record['root_cause']} " + "lands on target, but the record's note names an unlisted " + "input too): regenerate only if a reviewer rules that " + "reason out", + "excluded_rechecked", + ) + continue + if on_target: + regenerated.add(k) + upstream = upstream_for(record, k, upstreams) + actions["regenerated_exclusions"].append( + { + "scenario_id": k[0], + "variable": k[1], + "alternative_value": float(record["alternative_value"]), + "tolerance": TARGET_TOL, + "upstream": upstream, + "basis": ( + f"An engine defect fixed upstream ({upstream}) is " + f"regenerated with the fix: {engine} gives " + f"{money(cell.new)}, within $1 of {target_text} " + f"({record['root_cause']}: {record['defect']})." + ), + "target": target_spec, + } + ) + hub = cell.audited_target + note = ( + "" + if hub in ("", "yes") + else f" The hub's table disagrees ({hub}); resolve it first." + ) + if k in first_rechecks: + note += ( + " It was rechecked on 2026-10-10: read that reason in " + "release 20261010's engine_upgrade revision." + ) + if upstream.strip().lower() in UNNAMED_UPSTREAM: + note += ( + " The record names no upstream fix; the builder " + "refuses the regeneration until upstream names it." + ) + call( + cell, + "PROPOSED", + f"REGENERATE ({source} engine defect {record['root_cause']}): " + f"lands on {money(cell.new)} vs target {money(target)}. " + "Confirm the fix is released upstream and name it in " + f"upstream ({upstream or 'empty'}).{note}", + "regenerated_exclusions", + ) + continue + reason = ( + f"Stays excluded on {engine}: the engine gives {money(cell.new)}, " + f"{money(abs(cell.new - target))} from the record's corrected " + f"value {money(target)}, so the defect ({record['root_cause']}) " + f"is not fixed in this release; the record keeps " + f"{money(kept_value)}." + ) + if cell.audited_target.startswith("pending"): + reason += ( + " The US hub's sweep marks it pending on more than one fix " + f"({cell.audited_target.removeprefix('pending: ')})." + ) + text = ( + f"RECHECK ({source} engine defect {record['root_cause']}, off " + f"target by {money(abs(cell.new - target))})" + ) + else: + reason = ( + f"Stays excluded on {engine}: the reference depends on " + f"{record_basis(record)} ({record['reason_code']}), which an " + f"engine release does not settle; the record keeps " + f"{money(kept_value)}, and {engine} gives {money(cell.new)}." + ) + text = f"RECHECK ({source} {record['reason_code']})" + earlier = first_rechecks.get(k) + if earlier is not None and abs(float(earlier[ver_key]) - cell.new) <= 1e-3: + reason += ( + " The value is 2.38.6's, rechecked on 2026-10-10 (release " + "20261010's engine_upgrade revision's excluded_outputs_rechecked)." + ) + actions["excluded_rechecked"].append( + {"scenario_id": k[0], "variable": k[1], "reason": reason} + ) + call( + cell, + "PROPOSED", + text + " - check the drafted reason", + "excluded_rechecked", + ) + continue + for matches, propose in RULES: + if matches(cell): + proposal = propose(cell, engine, date) + actions[proposal["action"]].append(proposal["entry"]) + text = proposal["call"] + if not precise: + text += ( + "; the value is the table's, rounded to cents: rerun with " + "--computed (the builder's computed.csv) before the build" + ) + call(cell, "PROPOSED", text, proposal["action"]) + break + else: + call( + cell, + "UNDECIDED", + "scored move no rule recognizes: approve it (cause, basis) or " + "exclude it (a complete record); the builder refuses it until then", + "review only", + ) + actions["kept_exclusions_from_release"] = [ + {"scenario_id": k[0], "variable": k[1]} for k in sorted(ruled - regenerated) + ] + for name in ( + "approved", + "regenerated_exclusions", + "new_exclusions", + "excluded_rechecked", + ): + actions[name].sort(key=key_of) + actions["review"] = calls + return actions, calls + + +def print_calls(calls: list[dict]) -> None: + print(f"{len(calls)} moved outputs; every call below needs a reviewer:") + for c in calls: + print( + f" [{c['decision']}] {c['scenario_id']} {c['state']} {c['variable']}: " + f"{c['previous']:,.2f} -> {c['new']:,.2f} ({c['status']}) -> " + f"{c['placed_in']}\n {c['call']}" + ) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--cells") + parser.add_argument("--computed") + parser.add_argument("--engine") + parser.add_argument("--date") + parser.add_argument("--evidence", help="a committed evidence file, repo-relative") + parser.add_argument("--upstreams", help="upstreams.json: the fix each regeneration names") + parser.add_argument("--out", required=True) + args = parser.parse_args(argv) + evidence = None + if args.evidence: + ref = {"path": args.evidence, "sha256": sha256(ROOT / args.evidence)} + evidence = {"ref": ref, "doc": load_evidence(ref)} + if not (args.cells or args.computed): + parser.error("pass --computed (a builder pass) or --cells (a hub table)") + engine_version = args.engine or engine_from_path(Path(args.cells)) + date = args.date or datetime.datetime.now(datetime.timezone.utc).date().isoformat() + base = load_base() + release, ruled, spec_sha = release_record(base.exclusions) + computed = read_computed(Path(args.computed)) if args.computed else None + if args.cells: + cells = read_cells(Path(args.cells), base.reference.values(), computed) + else: + cells = cells_from_computed( + Path(args.computed), + base.reference.values(), + {key_of(r): r for r in release["exclusions"]}, + {key_of(r) for r in base.exclusions["exclusions"]}, + ) + actions, calls = draft_actions( + cells, + release, + ruled, + {key_of(r) for r in base.exclusions["exclusions"]}, + first_upgrade_rechecks(base.meta), + engine_version=engine_version, + date=date, + spec_sha256=spec_sha, + precise=computed is not None, + evidence=evidence, + upstreams=json.loads(Path(args.upstreams).read_text()) if args.upstreams else None, + ) + Path(args.out).write_text(json.dumps(actions, indent=2) + "\n") + print_calls(calls) + print(f"DRAFT -> {args.out} (draft: true; the builder refuses it until reviewed)") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json b/reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json new file mode 100644 index 00000000..9c72d017 --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json @@ -0,0 +1,116 @@ +{ + "kind": "regeneration_evidence", + "engine": "policyengine-us 2.38.6", + "policyengine_core": "3.32.29", + "computed_at_utc": "2026-10-12T23:15:29.638752+00:00", + "builder_sha256": "be5bdb265b22665055b844e6da92dbd391b31b411fe2f73f6ce60e856d9e0646", + "harness_sha256": "f4c7783453a473a1bedcf64ef554c3a2fc29f9d663626948d920ed3ef2ea2b69", + "conventions": "reference_audit/2026-09-28/fixes/latest_final.py", + "base_commit": "5a8164a001efb27fa55f47fe7ea26666a0de31f8", + "modules": { + "r02_ira_219g.py": "295a0b975ced8964fc1b2c5d52c47bce177eed399f49735eafb88441e7561b84", + "r02_ira_219g_v2.py": "937514fb1199c4ca172dfc5252db7f2483adade193922e6e5283d0b6ab4c4c8c", + "r06_wi_act15_before_refundable_v2.py": "1128ee23524a87bf53facee969833f99e28b731534381732773d801d71e29ada", + "r07_idaho_health_premiums_v2.py": "af9b7cc6dbd25b1e60b547cc1bee167527dbf7faee4dc88512c4c28416808222", + "r08_eitc_earned_income_deferrals_v2.py": "ba57e9e15f4a48a9e9aa4a916ed067246ecee28df64fc76f4633da044cf8bd9b", + "r11_ca_itemized_conformity_v2.py": "2bcb8ba6694c053f3c9a8509b96bfee4976ba00e3fb676c0c504e06c2d6c046d", + "r30_snap_heat_and_eat_sua.py": "11072570d931a4e77e50216e4f7c926c83749e3d437afaa8f34442d102cd6b82", + "r32_wi_capital_gain_distributions.py": "1b25c10c4a6139e0b9e2a1a6eb764932c91bb3036d5ff69986d7d6469c10f823", + "r33_ca_educator_expense_addback.py": "4545d27858790ed537be056b9d1ae3e0a7c39389486ff8af1e273106cb58efad" + }, + "items": [ + { + "scenario_id": "scenario_007", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r07_idaho_health_premiums_v2.py" + ], + "engine_value": 771.5570068359375, + "corrected_value": 661.3170166015625 + }, + { + "scenario_id": "scenario_042", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r06_wi_act15_before_refundable_v2.py", + "r32_wi_capital_gain_distributions.py" + ], + "engine_value": 464.18280029296875, + "corrected_value": 0.0 + }, + { + "scenario_id": "scenario_053", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r07_idaho_health_premiums_v2.py" + ], + "engine_value": 2451.05712890625, + "corrected_value": 2186.05712890625 + }, + { + "scenario_id": "scenario_080", + "variable": "snap", + "modules": [ + "r30_snap_heat_and_eat_sua.py" + ], + "engine_value": 3576.0, + "corrected_value": 3240.0 + }, + { + "scenario_id": "scenario_091", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r32_wi_capital_gain_distributions.py" + ], + "engine_value": 895.8131103515625, + "corrected_value": 878.5157470703125 + }, + { + "scenario_id": "scenario_099", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r02_ira_219g_v2.py", + "r11_ca_itemized_conformity_v2.py", + "r33_ca_educator_expense_addback.py" + ], + "engine_value": 4588.48388671875, + "corrected_value": 4615.48388671875 + }, + { + "scenario_id": "scenario_100", + "variable": "federal_refundable_credits", + "modules": [ + "r08_eitc_earned_income_deferrals_v2.py" + ], + "engine_value": 2878.0966796875, + "corrected_value": 822.2882690429688 + }, + { + "scenario_id": "scenario_100", + "variable": "state_refundable_credits", + "modules": [ + "r08_eitc_earned_income_deferrals_v2.py" + ], + "engine_value": 473.1777038574219, + "corrected_value": 164.45765686035156 + }, + { + "scenario_id": "scenario_119", + "variable": "federal_refundable_credits", + "modules": [ + "r08_eitc_earned_income_deferrals_v2.py" + ], + "engine_value": 2854.0087890625, + "corrected_value": 3366.07373046875 + }, + { + "scenario_id": "scenario_119", + "variable": "state_income_tax_before_refundable_credits", + "modules": [ + "r08_eitc_earned_income_deferrals_v2.py" + ], + "engine_value": 1859.0943603515625, + "corrected_value": 1756.681396484375 + } + ] +} diff --git a/reference_audit/2026-10-12-engine-upgrade/scripts/build_references_upgrade.py b/reference_audit/2026-10-12-engine-upgrade/scripts/build_references_upgrade.py new file mode 100644 index 00000000..4cecd77b --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/scripts/build_references_upgrade.py @@ -0,0 +1,1715 @@ +"""Rebuild PolicyBench's US references on the newest policyengine-us (the third +engine upgrade, after release dashboard-data-20261010). + +Adapted from reference_audit/2026-10-09-engine-upgrade/scripts/ +build_references_upgrade.py, which moved the references from policyengine-us +2.15.17 to 2.38.6 for release 20261010; that builder generalized +reference_audit/2026-09-28/scripts/build_references_latest.py (PR #182). + +Rulings: +- Max, 2026-09-28: "we should be using the latest pe for this always!" +- Max, 2026-10-09: "yes i want to wait for hte fixed engine". A cell the new + engine fixes is regenerated and its exclusion record removed (the #178 + precedent, cb312fd7). +- Max, 2026-10-12, on the release that adds Muse Wonder, of the engine fixes + release 20261010 left excluded: "If any have shipped, regenerate their cells + instead of excluding them." +- The release rules no exclusion (docs/musewonder/spec.json lists none). +- Rule 5: an excluded output keeps the value it was decided on; an excluded + output that moves gets an excluded_rechecked entry. + +Base (read from git, never from the checkout, whose snapshot holds whatever was +last installed): release 20261010 at BASE_COMMIT, i.e. 2.38.6 references and the +58-record exclusion record, each file pinned by sha256. The release record this +script edits comes from scripts/finish_musewonder.py's build_release_exclusions, +so it is byte-for-byte the one the Muse Wonder driver installs (release +20261010's, since nothing is ruled). The original v1.1 frozen values (each +change's "frozen") come from FROZEN_COMMIT. + +Method: every output is computed on the installed policyengine-us with +reference_audit/2026-09-28/fixes/latest_final.py (the pre-freeze conventions and +the Maryland local-scope adapter), imported in place from --fixes-dir, with the +households built by reference_audit/2026-09-28/scripts/sweep.py's +build_situation. Every fix module's sha256 must equal the one the 2026-09-29 +engine_upgrade revision records; latest_final.py, sweep.py and the sales tax +tables (r19_irs_sales_tax_2025.json, which latest_c_irs_sales_tax_2025.py reads +from its own directory) must each equal the bytes committed at BASE_COMMIT (the +tables as reference_audit/2026-09-22/fixes holds them). + +A move is a change beyond the exact-match tolerance +(policybench.paper_results.moves_beyond_tolerance: more than $1 for an amount, +any change for a 0/1 flag). A scored output takes the computed value; one that +changes by $1 or less (more than EPS) is recorded as engine_upgrade_within_1. + +ACTIONS FILE (JSON; every list optional except where a move needs it): + + engine "policyengine-us X.Y.Z"; must equal the installed release. + previous_engine "policyengine-us 2.38.6"; must equal the base sidecar's + last engine_upgrade engine_version. + date UTC day of the build (YYYY-MM-DD): the revision's date, the + derivation's, and each new record's decided_on. The sidecar's + regenerated_at_utc must fall on it. + release_spec_sha256 sha256 of docs/musewonder/spec.json the actions were drafted + against. + draft true for actions_from_cells.py's output; the builder + refuses a draft unless --allow-draft (rehearsals only). + review, note free text for reviewers; ignored. + approved [{scenario_id, variable, value, cause, basis}]: scored + outputs allowed to move; the engine must give value within + APPROVED_TOL (half a cent). + regenerated_exclusions [{scenario_id, variable, alternative_value, + tolerance, upstream, basis, target}]: excluded outputs (in + 20261010's record or the release's ruled records) whose + reference_engine_defect the new engine fixes. alternative_value + must be the record's; tolerance is at most $1; upstream must + name the fix ("to be filed" does not). The engine must land + within the tolerance of the audited corrected value (a flag + must equal it), which target says how to know: + {"kind": "record"} (the default): the record's + alternative_value. Right when nothing else in the engine + has moved the output since the record was decided. + {"kind": "fix_modules", "modules": [file, ...], + "evidence": {"path", "sha256"}}: the record's audited fix + modules (reference_audit/2026-09-22/fixes, the bytes + committed at BASE_COMMIT), on a recent engine that still + has the defect. The evidence file (--evidence-request + mode, on that engine) gives the output there without and + with the modules; they must move it beyond the tolerance + (the defect was there), and the corrected value is the + target. On the new engine the builder applies the modules + again: they must move it by no more than the tolerance + (nothing is left to fix). A record decided on an older + engine needs this kind once other engine changes have + moved the output, because its alternative_value no longer + includes them. + The record is removed and the output scored at the engine + value. + new_exclusions [complete exclusion record]: scored outputs the new engine + moves onto an input the prompt does not state (for example + Indiana county). frozen_value must be the engine value + (within 1e-3), engine_version the new engine, decided_on the + date, and the alternative must differ beyond the tolerance. + excluded_rechecked [{scenario_id, variable, reason}]: excluded outputs that + move but stay excluded, keeping the decided value. + kept_exclusions_from_release [{scenario_id, variable}]: the release's ruled + outputs that stay excluded. Together with the ruled outputs + listed in regenerated_exclusions they must be exactly the + spec's ruled outputs. A kept output that moves is also + listed in excluded_rechecked, the one pair of lists that + may share an output. + +The builder refuses, listing every problem with exact engine values: an +unexplained move (scored or excluded), an approved output that does not move or +lands elsewhere, a regenerated output off its target or not an engine defect, a +new exclusion on an output that does not move, a rechecked output that does not +move, an entry on an unknown output, and an output named by two actions. + +It writes to --out-dir: + reference_outputs.csv 20261010's rows byte for byte, except each changed row, + whose value field is rewritten. + reference_exclusions.json the release record (20261010's 58; nothing ruled) + less the regenerated records, with the new exclusions + before the trailing 2026-09-29 audit record and one + sentence appended to the derivation naming the upgrade. + reference_outputs.csv.meta.json the base sidecar with the US bundle, csv + sha256 and regenerated_at_utc updated and a third + engine_upgrade revision appended; every earlier revision + is kept byte for byte. + reference_traces.json engine traces of every changed output. + computed.csv (also with --computed-csv, written before any refusal) + every output on the new engine, in sweep.py's columns. + +Run (PYTHONPATH is the checkout; the fixes directory needs the sales tax tables): + + PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ + /bin/python \\ + reference_audit/2026-10-12-engine-upgrade/scripts/build_references_upgrade.py \\ + --actions --out-dir [--fixes-dir ] [--allow-draft] + +EVIDENCE MODE (on a pre-fix engine, for "fix_modules" targets): + + ... build_references_upgrade.py --evidence-request \\ + --evidence-out [--fixes-dir ] + +The request is {"items": [{scenario_id, variable, modules: [file, ...]}]}. The +evidence file names the installed engine and every module's sha256, and gives +each item's engine_value (the pinned conventions alone) and corrected_value +(the conventions plus the modules, applied after them). +""" + +from __future__ import annotations + +import argparse +import copy +import csv +import datetime +import hashlib +import importlib.util +import io +import json +import math +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +RUN = "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +# The merge of PR #208 on main, whose tree holds release 20261010. +BASE_COMMIT = "5a8164a001efb27fa55f47fe7ea26666a0de31f8" +BASE_SHA256 = { + "reference_outputs.csv": ( + "fdd17a57092df4a9111652c85609c767bdbd05b6e32602dd8201b0b74bbbf371" + ), + "reference_outputs.csv.meta.json": ( + "d534cfeacf7f8a8b7f6fca477adbb02c5bb52b226778d63cc10fd117ec70c54b" + ), + "reference_exclusions.json": ( + "089e01e1f699aa19f2171e1ac5b6dc2ce775f9898c94420e6a0b06ff5f773626" + ), + "scenarios.csv": ( + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a" + ), +} +# Re-freeze on the 24-model board (#121): the snapshot's last commit holding the +# v1.1 frozen references (policyengine-us 1.755.4), the bundle the 2026-09-29 +# builder read as "frozen". +FROZEN_COMMIT = "f108a959d91567c050a99d4d1d583904b3d5a809" +FROZEN_SHA256 = "b9136a15e285f9c02ba78bee854b8a3af180e280485512642c829e8ffd7d2368" +PREVIOUS_ENGINE = "policyengine-us 2.38.6" +FIXES_REL = "reference_audit/2026-09-28/fixes" +FIXES = ROOT / FIXES_REL +HARNESS_REL = "reference_audit/2026-09-28/scripts/sweep.py" +HARNESS = ROOT / HARNESS_REL +FIX_ENTRY = "latest_final" +SALES_TAX_TABLES = "r19_irs_sales_tax_2025.json" +SALES_TAX_REL = f"reference_audit/2026-09-22/fixes/{SALES_TAX_TABLES}" +SALES_TAX_SOURCE = ROOT / SALES_TAX_REL +# The audited root-cause fix modules (2026-09-22): a "fix_modules" target +# applies them after the conventions. +AUDIT_FIXES_REL = "reference_audit/2026-09-22/fixes" +AUDIT_FIXES = ROOT / AUDIT_FIXES_REL +# This audit's own fix modules, for defects found after 2026-09-22 (r33, the +# California educator-expense addback). A module that is not committed at +# BASE_COMMIT under AUDIT_FIXES_REL is read from here, at the bytes committed +# at HEAD: commit it before the evidence or the build runs. A target names +# such a module's path (fix_location); scripts/freeze_snapshot.py's +# verify_regeneration_target_pins holds the published pin to it. +LOCAL_FIXES_REL = "reference_audit/2026-10-12-engine-upgrade/fixes" +LOCAL_FIXES = ROOT / LOCAL_FIXES_REL +EVIDENCE_KIND = "regeneration_evidence" +# A fix module loads a sibling as Path(__file__).with_name(""); those +# siblings run too, so they are pinned with it (module_dependencies, from +# policybench.fix_module_closure, which refuses any other way of loading one). +TARGET_RECORD = "record" +TARGET_FIX_MODULES = "fix_modules" +YEAR = 2026 +EPS = 1e-6 +FROZEN_TOL = 1e-3 +APPROVED_TOL = 0.005 +MAX_REGENERATION_TOL = 1.0 +# The trailing record of 20261010's exclusion record: the 2026-09-29 audit's +# exclusion, which tests/test_reference_upgrade.py requires last. +AUDIT_TAIL = ("scenario_023", "head_medicaid_eligible") +ENGINE_DEFECT = "reference_engine_defect" +# A regeneration claims a released upstream fix and must name it. +UNNAMED_UPSTREAM = frozenset({"", "to be filed", "none", "unknown"}) +ACTION_KEYS = frozenset( + { + "engine", + "previous_engine", + "date", + "release_spec_sha256", + "draft", + "review", + "note", + "approved", + "regenerated_exclusions", + "new_exclusions", + "excluded_rechecked", + "kept_exclusions_from_release", + } +) +RULE = ( + "A scored reference follows from the stated facts and from law published before " + "the 2026-07-03 reference freeze." +) +ENGINE_RULE = ( + "References come from the newest policyengine-us release when PolicyBench begins " + "the reference sweep; at publication PolicyBench checks that the newest release " + "gives the same values." +) +UPGRADE_RULE = ( + "Max, 2026-10-09, on waiting for the release that fixes the engine defects " + "behind PolicyBench's exclusions: 'yes i want to wait for hte fixed engine'; " + "and 2026-10-12, of the fixes release 20261010 left excluded: 'If any have " + "shipped, regenerate their cells instead of excluding them.' An " + "excluded output whose engine defect the new release fixes, landing on its " + "audited corrected value, is regenerated and its exclusion record removed; every " + "other excluded output keeps the value it was decided on." +) +NUMBER_WORDS = { + 1: "one", + 2: "two", + 3: "three", + 4: "four", + 5: "five", + 6: "six", + 7: "seven", + 8: "eight", + 9: "nine", + 10: "ten", +} + +Key = tuple[str, str] + + +class Refusal(SystemExit): + """The builder refuses to write references.""" + + +def sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def sha256(path: Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def dump_record(value: dict) -> str: + """The serialization of the sidecar and the exclusion record.""" + return json.dumps(value, indent=2) + "\n" + + +def words(n: int) -> str: + return NUMBER_WORDS.get(n, str(n)) + + +def key_of(item: dict) -> Key: + return (str(item["scenario_id"]), str(item["variable"])) + + +def engine_number(engine: str) -> str: + prefix = "policyengine-us " + if not engine.startswith(prefix): + raise Refusal(f"engine must read 'policyengine-us X.Y.Z', not {engine!r}") + return engine.removeprefix(prefix) + + +def engine_older(older: str, newer: str) -> bool: + """Whether one 'policyengine-us X.Y.Z' release precedes another.""" + + def parts(engine: str) -> tuple[int, ...]: + return tuple(int(p) for p in engine_number(engine).split(".")) + + return parts(older) < parts(newer) + + +def beyond(variable: str, before: float, after: float) -> bool: + """A move beyond the exact-match tolerance, as the paper counts it.""" + from policybench.paper_results import moves_beyond_tolerance + + return moves_beyond_tolerance(variable, before, after) + + +# --- Base ------------------------------------------------------------------ + + +def git_blob(commit: str, path: str, root: Path = ROOT) -> bytes: + result = subprocess.run( + ["git", "-C", str(root), "show", f"{commit}:{path}"], capture_output=True + ) + if result.returncode: + raise Refusal( + f"cannot read {path} at {commit[:12]} (fetch full history): " + f"{result.stderr.decode().strip()}" + ) + return result.stdout + + +@dataclass +class Row: + key: Key + fields: list[str] + raw: str + + @property + def value(self) -> float: + return float(self.fields[2]) + + +@dataclass +class ReferenceCsv: + header_raw: str + header: list[str] + rows: list[Row] + + def values(self) -> dict[Key, float]: + return {row.key: row.value for row in self.rows} + + +def physical_lines(text: str) -> list[str]: + """Split on '\\n' only, each piece keeping its terminator.""" + lines = text.split("\n") + out = [line + "\n" for line in lines[:-1]] + if lines[-1]: + out.append(lines[-1]) + return out + + +def parse_reference_csv(text: str) -> ReferenceCsv: + """Rows with their raw text, so an unchanged row is written byte for byte.""" + lines = physical_lines(text) + reader = csv.reader(iter(lines)) + records, used = [], 0 + for fields in reader: + records.append((fields, "".join(lines[used : reader.line_num]))) + used = reader.line_num + (header, header_raw), body = records[0], records[1:] + if header[:3] != ["scenario_id", "variable", "value"]: + raise Refusal(f"unexpected reference header {header}") + rows = [Row((f[0], f[1]), f, raw) for f, raw in body] + if len({row.key for row in rows}) != len(rows): + raise Refusal("the reference CSV lists an output twice") + return ReferenceCsv(header_raw, header, rows) + + +def format_row(fields: list[str]) -> str: + buffer = io.StringIO() + csv.writer(buffer, lineterminator="\n").writerow(fields) + return buffer.getvalue() + + +def render_reference_csv(reference: ReferenceCsv, new_values: dict[Key, float]) -> str: + """The base CSV with each output in new_values rewritten at that value.""" + parts = [reference.header_raw] + for row in reference.rows: + if row.key in new_values: + fields = list(row.fields) + fields[2] = repr(float(new_values[row.key])) + raw = format_row(fields) + if not row.raw.endswith("\n"): + raw = raw.removesuffix("\n") + parts.append(raw) + else: + parts.append(row.raw) + return "".join(parts) + + +@dataclass +class Base: + reference: ReferenceCsv + meta: dict + exclusions: dict + scenarios_csv: str + frozen: dict[Key, float] + + +def load_base(root: Path = ROOT) -> Base: + """Release 20261010's references at BASE_COMMIT, each file pinned.""" + blobs = {} + for name, pin in BASE_SHA256.items(): + raw = git_blob(BASE_COMMIT, f"{RUN}/{name}", root) + if sha256_bytes(raw) != pin: + raise Refusal(f"{name} at {BASE_COMMIT[:12]} does not match its pin") + blobs[name] = raw.decode() + frozen_raw = git_blob(FROZEN_COMMIT, f"{RUN}/reference_outputs.csv", root) + if sha256_bytes(frozen_raw) != FROZEN_SHA256: + raise Refusal("the v1.1 frozen references do not match their pin") + return Base( + reference=parse_reference_csv(blobs["reference_outputs.csv"]), + meta=json.loads(blobs["reference_outputs.csv.meta.json"]), + exclusions=json.loads(blobs["reference_exclusions.json"]), + scenarios_csv=blobs["scenarios.csv"], + frozen=parse_reference_csv(frozen_raw.decode()).values(), + ) + + +def release_record(base_exclusions: dict) -> tuple[dict, set[Key], str]: + """The release's exclusion record, as scripts/finish_musewonder.py builds it + from the spec (release 20261010's 58, since nothing is ruled); the ruled + outputs; and the spec's sha256.""" + path = ROOT / "scripts" / "finish_musewonder.py" + spec = importlib.util.spec_from_file_location("finish_musewonder_for_upgrade", path) + driver = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = driver + spec.loader.exec_module(driver) + release_spec = driver.load_spec() + doc = driver.build_release_exclusions(copy.deepcopy(base_exclusions), release_spec) + ruled = {driver.spec_key(r) for r in driver.spec_records(release_spec)} + return doc, ruled, sha256(ROOT / driver.SPEC_PATH) + + +# --- The plan ---------------------------------------------------------------- + + +@dataclass +class Plan: + problems: list[str] = field(default_factory=list) + new_values: dict[Key, float] = field(default_factory=dict) + changed: list[dict] = field(default_factory=list) + within: list[dict] = field(default_factory=list) + rechecked: list[dict] = field(default_factory=list) + regenerated: list[dict] = field(default_factory=list) + new_records: list[dict] = field(default_factory=list) + kept: list[Key] = field(default_factory=list) + + +def _entries(actions: dict, name: str) -> dict[Key, dict]: + entries: dict[Key, dict] = {} + for item in actions.get(name, []): + k = key_of(item) + if k in entries: + raise Refusal(f"{name} lists {k} twice") + entries[k] = item + return entries + + +def validate_actions(actions: dict, *, allow_draft: bool = False) -> None: + unknown = set(actions) - ACTION_KEYS + if unknown: + raise Refusal(f"unknown actions keys {sorted(unknown)}") + for name in ("engine", "previous_engine", "date"): + if not isinstance(actions.get(name), str) or not actions[name]: + raise Refusal(f"actions must name {name}") + datetime.date.fromisoformat(actions["date"]) + engine_number(actions["engine"]) + if actions.get("draft") and not allow_draft: + raise Refusal( + "the actions file is a draft (actions_from_cells.py output): review " + "every judgment call and set draft to false, or pass --allow-draft " + "for a rehearsal" + ) + lists = ( + "approved", + "regenerated_exclusions", + "new_exclusions", + "excluded_rechecked", + "kept_exclusions_from_release", + ) + # A kept ruled output that moves is also rechecked (rule 5): kept declares + # it stays excluded, rechecked records its move. No other pair may share an + # output. + allowed = {frozenset({"excluded_rechecked", "kept_exclusions_from_release"})} + seen: dict[Key, str] = {} + for name in lists: + for k in _entries(actions, name): + if k in seen and frozenset({seen[k], name}) not in allowed: + raise Refusal(f"{k} is in both {seen[k]} and {name}") + seen.setdefault(k, name) + + +def target_modules(entry: dict) -> tuple[str, ...]: + """The fix modules a regeneration's target names (none for a record).""" + target = entry.get("target") or {} + modules = target.get("modules") if isinstance(target, dict) else None + if isinstance(modules, list) and all(isinstance(m, str) for m in modules): + return tuple(modules) + return () + + +def regeneration_target( + k: Key, + entry: dict, + record: dict, + value: float, + tolerance: float, + *, + engine: str, + evidence: dict[str, dict], + module_pins: dict[str, str], + module_values: dict[tuple[Key, tuple[str, ...]], float], + module_deps: dict[str, list[str]] | None = None, +) -> tuple[dict | None, list[str]]: + """The audited corrected value a regenerated output must land on, and how + it is known (the entry's target), or the problems that stop it. Pure.""" + target = entry.get("target", {"kind": TARGET_RECORD}) + if not isinstance(target, dict): + return None, [f"regenerated {k}: target is not an object"] + kind = target.get("kind") + if kind == TARGET_RECORD: + if set(target) != {"kind"}: + return None, [f"regenerated {k}: a record target takes no other fields"] + return { + "kind": TARGET_RECORD, + "value": float(record["alternative_value"]), + "engine": record["engine_version"], + }, [] + if kind != TARGET_FIX_MODULES: + return None, [f"regenerated {k}: unknown target kind {kind!r}"] + modules = target_modules(entry) + problems = [] + if set(target) != {"kind", "modules", "evidence"}: + problems.append(f"regenerated {k}: a fix_modules target takes kind, modules " + "and evidence") + if not modules or len(set(modules)) != len(modules): + problems.append(f"regenerated {k}: the target names no modules, or one twice") + unpinned = [m for m in modules if m not in module_pins] + if unpinned: + problems.append(f"regenerated {k}: unknown fix modules {unpinned}") + ref = target.get("evidence") + doc = evidence.get(ref.get("path")) if isinstance(ref, dict) else None + if doc is None: + problems.append(f"regenerated {k}: the target's evidence is not loaded") + if problems: + return None, problems + variable = k[1] + if not engine_older(doc["engine"], engine): + problems.append( + f"regenerated {k}: the evidence is computed on {doc['engine']}, not an " + f"engine older than {engine}; it must come from one that still has the " + "defect" + ) + stale = [m for m in modules if doc["modules"].get(m) != module_pins[m]] + if stale: + problems.append(f"regenerated {k}: the evidence ran other bytes of {stale}") + items = [ + item + for item in doc["items"] + if key_of(item) == k and tuple(item["modules"]) == modules + ] + if len(items) != 1: + problems.append( + f"regenerated {k}: the evidence has {len(items)} items for {list(modules)}" + ) + return None, problems + item = items[0] + before, corrected = float(item["engine_value"]), float(item["corrected_value"]) + dependencies = [] + for m in modules: + for d in (module_deps or {}).get(m, []): + if d not in modules and d not in dependencies: + dependencies.append(d) + unpinned = [ + d for d in dependencies if d not in module_pins or doc["modules"].get(d) != module_pins[d] + ] + if unpinned: + problems.append( + f"regenerated {k}: the evidence does not pin the siblings its modules " + f"load, as committed: {unpinned}" + ) + if not (math.isfinite(before) and math.isfinite(corrected)): + problems.append(f"regenerated {k}: the evidence's values are not finite") + return None, problems + if not beyond(variable, before, corrected): + problems.append( + f"regenerated {k}: on {doc['engine']} the modules move it only " + f"{before!r} -> {corrected!r}, so the evidence shows no defect" + ) + after = module_values.get((k, modules)) + if after is None: + problems.append(f"regenerated {k}: the modules were not applied on {engine}") + elif not math.isfinite(after) or abs(after - value) > tolerance or beyond( + variable, value, after + ): + problems.append( + f"regenerated {k}: the audited fix still moves it on {engine}: " + f"{value!r} -> {after!r}" + ) + if problems: + return None, problems + return { + "kind": TARGET_FIX_MODULES, + "value": corrected, + "engine": doc["engine"], + "engine_value": before, + "modules": [module_pin(m, module_pins) for m in modules], + "dependencies": [module_pin(d, module_pins) for d in dependencies], + "evidence": ref["path"], + "evidence_sha256": ref["sha256"], + "value_with_modules": after, + }, [] + + +def module_pin(name: str, module_pins: dict[str, str]) -> dict: + """A target's pin of one module: its name and sha256, and its path when it + is this audit's own (LOCAL_FIXES_REL), where the freeze finds it.""" + pin = {"module": name, "sha256": module_pins[name]} + if is_local_fix(name): + pin["path"] = f"{LOCAL_FIXES_REL}/{name}" + return pin + + +def plan_upgrade( + base: Base, + release: dict, + ruled: set[Key], + actions: dict, + computed: dict[Key, float], + *, + evidence: dict[str, dict] | None = None, + module_pins: dict[str, str] | None = None, + module_values: dict[tuple[Key, tuple[str, ...]], float] | None = None, + module_deps: dict[str, list[str]] | None = None, +) -> Plan: + """Decide every output's value and record; never raise on a move, collect + every problem instead. Pure: no engine, no files. A "fix_modules" target + reads its evidence (loaded and pinned by the caller), the modules' pins and + their values on the new engine from the keyword arguments.""" + evidence = evidence or {} + module_pins = module_pins or {} + module_values = module_values or {} + plan = Plan() + engine = actions["engine"] + date = actions["date"] + board = base.reference.values() + records = {key_of(r): r for r in release["exclusions"]} + excluded = set(records) + base_excluded = {key_of(r) for r in base.exclusions["exclusions"]} + if not ruled <= excluded or excluded - ruled != base_excluded: + plan.problems.append("the release record is not 20261010's plus the ruled") + approved = _entries(actions, "approved") + regenerated = _entries(actions, "regenerated_exclusions") + added = _entries(actions, "new_exclusions") + rechecked = _entries(actions, "excluded_rechecked") + kept = _entries(actions, "kept_exclusions_from_release") + if set(computed) != set(board): + missing = sorted(set(board) - set(computed))[:5] + extra = sorted(set(computed) - set(board))[:5] + plan.problems.append( + f"computed outputs differ: missing {missing} extra {extra}" + ) + for name, entries in ( + ("approved", approved), + ("regenerated_exclusions", regenerated), + ("new_exclusions", added), + ("excluded_rechecked", rechecked), + ("kept_exclusions_from_release", kept), + ): + for k in entries: + if k not in board: + plan.problems.append(f"{name} names {k}, which is not a reference") + if actions["previous_engine"] != PREVIOUS_ENGINE: + plan.problems.append( + f"previous_engine is {actions['previous_engine']}, not {PREVIOUS_ENGINE}" + ) + regenerated_ruled = {k for k in regenerated if k in ruled} + if set(kept) | regenerated_ruled != ruled: + missing = sorted(ruled - set(kept) - regenerated_ruled) + plan.problems.append( + "kept_exclusions_from_release and the regenerated ruled outputs must be " + f"the spec's ruled outputs: missing {missing}, extra " + f"{sorted(set(kept) - ruled)}" + ) + plan.kept = sorted(k for k in kept if k in ruled) + + def change(k: Key, value: float, cause: str, basis: str) -> dict: + return { + "scenario_id": k[0], + "variable": k[1], + "frozen": float(base.frozen[k]), + "previous": board[k], + "regenerated": value, + "cause": cause, + "basis": basis, + } + + within_basis = ( + f"Moves by $1 or less on {engine}; no score changes at the exact-match " + "tolerance." + ) + for row in base.reference.rows: + k = row.key + if k not in computed: + continue + value, old = float(computed[k]), board[k] + if not math.isfinite(value): + plan.problems.append(f"{k}: the engine gives a non-finite value {value!r}") + continue + moves = beyond(k[1], old, value) + if k in excluded: + for name, entries in (("approved", approved), ("new_exclusions", added)): + if k in entries: + plan.problems.append(f"{name} names {k}, which is excluded") + record = records[k] + if k in regenerated: + entry = regenerated[k] + alternative = float(record["alternative_value"]) + tolerance = float(entry.get("tolerance", -1)) + if not math.isfinite(tolerance): + tolerance = -1.0 + if record["reason_code"] != ENGINE_DEFECT: + plan.problems.append( + f"regenerated {k} is {record['reason_code']}, not an engine " + "defect" + ) + continue + if abs(float(entry["alternative_value"]) - alternative) > EPS: + plan.problems.append( + f"regenerated {k}: alternative_value " + f"{entry['alternative_value']} is not the record's " + f"{alternative}" + ) + continue + if not 0 <= tolerance <= MAX_REGENERATION_TOL: + plan.problems.append( + f"regenerated {k}: tolerance {tolerance} outside [0, " + f"{MAX_REGENERATION_TOL}]" + ) + continue + target, problems = regeneration_target( + k, + entry, + record, + value, + tolerance, + engine=engine, + evidence=evidence, + module_pins=module_pins, + module_values=module_values, + module_deps=module_deps, + ) + if problems: + plan.problems.extend(problems) + continue + # A flag must equal its target; an amount must land within the + # tolerance, which is at most the exact-match $1. + aim = target["value"] + if abs(value - aim) > tolerance or beyond(k[1], aim, value): + plan.problems.append( + f"regenerated {k} misses its audited target: engine " + f"{value!r}, target {aim!r} +/- {tolerance} " + f"({target['kind']})" + ) + continue + if not str(entry.get("basis", "")).strip(): + plan.problems.append(f"regenerated {k} has no basis") + continue + upstream = str(entry.get("upstream", "")).strip() + if upstream.lower() in UNNAMED_UPSTREAM: + plan.problems.append( + f"regenerated {k} names no upstream fix ({upstream!r})" + ) + continue + if abs(value - old) > EPS: + plan.new_values[k] = value + plan.changed.append( + change(k, value, "regenerated_upstream_fix", entry["basis"]) + ) + plan.regenerated.append( + { + "scenario_id": k[0], + "variable": k[1], + "root_cause": record["root_cause"], + "decided_on": record["decided_on"], + "kept_value": old, + f"value_on_{engine_number(engine).replace('.', '_')}": value, + "audited_alternative_value": alternative, + "target": target, + "regenerated": value, + "tolerance": tolerance, + "upstream": entry["upstream"], + "basis": entry["basis"], + "record": copy.deepcopy(record), + } + ) + continue + # Stays excluded: keeps the value it was decided on (rule 5). + if moves: + if k not in rechecked: + plan.problems.append( + f"excluded output moved without a recheck: {k}: " + f"{old!r} -> {value!r}" + ) + elif not str(rechecked[k].get("reason", "")).strip(): + plan.problems.append(f"rechecked {k} has no reason") + else: + plan.rechecked.append( + { + "scenario_id": k[0], + "variable": k[1], + "kept_value": old, + f"value_on_{engine_number(engine).replace('.', '_')}": ( + value + ), + "reason": rechecked[k]["reason"], + } + ) + elif k in rechecked: + plan.problems.append( + f"rechecked {k} does not move: {old!r} -> {value!r}" + ) + continue + # Scored in the base and in the release. + for name, entries in ( + ("regenerated_exclusions", regenerated), + ("excluded_rechecked", rechecked), + ("kept_exclusions_from_release", kept), + ): + if k in entries: + plan.problems.append(f"{name} names {k}, which is scored") + if k in added: + record = added[k] + problems = new_exclusion_problems(record, value, old, engine, date) + if problems: + plan.problems.extend(f"new exclusion {k}: {p}" for p in problems) + continue + plan.new_values[k] = value + plan.new_records.append(copy.deepcopy(record)) + plan.changed.append( + change( + k, + value, + f"excluded_{record['reason_code']}", + f"Newly excluded from scoring on {record['decided_on']} " + "(reference_exclusions.json): " + + str( + record.get("unlisted_input") + or record.get("root_cause") + or record["alternative_reading"] + ), + ) + ) + continue + if k in approved: + entry = approved[k] + if not moves: + plan.problems.append( + f"approved {k} does not move: {old!r} -> {value!r}" + ) + continue + if not math.isfinite(float(entry["value"])) or ( + abs(value - float(entry["value"])) > APPROVED_TOL + ): + plan.problems.append( + f"approved {k}: engine {value!r} != approved {entry['value']!r}" + ) + continue + if ( + not str(entry.get("cause", "")).strip() + or not str(entry.get("basis", "")).strip() + ): + plan.problems.append(f"approved {k} needs a cause and a basis") + continue + plan.new_values[k] = value + plan.changed.append(change(k, value, entry["cause"], entry["basis"])) + continue + if moves: + plan.problems.append(f"unreviewed move {k}: {old!r} -> {value!r}") + continue + if abs(value - old) > EPS: + plan.new_values[k] = value + plan.within.append( + change(k, value, "engine_upgrade_within_1", within_basis) + ) + plan.new_records.sort(key=key_of) + return plan + + +def new_exclusion_problems( + record: dict, value: float, old: float, engine: str, date: str +) -> list[str]: + from policybench.reference_exclusions import ( + REASON_CODES, + REQUIRED_FIELDS_BY_REASON, + ) + + variable = record["variable"] + if record.get("reason_code") not in REASON_CODES: + return [f"unknown reason_code {record.get('reason_code')!r}"] + missing = [ + name + for name in REQUIRED_FIELDS_BY_REASON[record["reason_code"]] + if record.get(name) in (None, "") + ] + if missing: + return [f"record missing {missing}"] + problems = [] + if not beyond(variable, old, value): + problems.append(f"the output does not move: {old!r} -> {value!r}") + if abs(float(record["frozen_value"]) - value) > FROZEN_TOL: + problems.append(f"frozen_value {record['frozen_value']!r} != engine {value!r}") + if not beyond( + variable, float(record["frozen_value"]), float(record["alternative_value"]) + ): + problems.append("the alternative is within the exact-match tolerance") + if record["engine_version"] != engine: + problems.append(f"engine_version {record['engine_version']!r} != {engine!r}") + if record["decided_on"] != date: + problems.append(f"decided_on {record['decided_on']!r} != {date!r}") + return problems + + +# --- Records ------------------------------------------------------------------- + + +def upgrade_sentence(plan: Plan, actions: dict) -> str: + """The derivation's sentence naming the upgrade.""" + engine, date = actions["engine"], actions["date"] + number = engine_number(engine) + parts = [ + f" On {date} the references moved from {actions['previous_engine']} to" + f" {engine} with the same pre-freeze conventions" + " (reference_audit/2026-10-12-engine-upgrade), and every output was" + " recomputed there." + ] + n = len(plan.regenerated) + if n == 1: + parts.append( + " The new engine fixes the defect behind one excluded output, which" + " lands within $1 of its audited corrected value, so its record was" + " removed and the output is scored again (the reference sidecar's third" + " engine_upgrade revision lists it under regenerated_exclusions)." + ) + elif n: + parts.append( + f" The new engine fixes the defects behind {words(n)} excluded outputs," + " each landing within $1 of its audited corrected value, so their" + " records were removed and the outputs are scored again (the reference" + " sidecar's third engine_upgrade revision lists them under" + " regenerated_exclusions)." + ) + n = len(plan.new_records) + if n == 1: + parts.append( + " One scored output the new engine moved was excluded that day; its" + f" record is computed on {number}." + ) + elif n: + parts.append( + f" {words(n).capitalize()} scored outputs the new engine moved were" + f" excluded that day; their records are computed on {number}." + ) + n = len(plan.rechecked) + if n == 1: + parts.append( + " Of the excluded outputs that stay excluded, one moved; it was" + " re-reviewed and keeps the value it was decided on" + " (excluded_outputs_rechecked)." + ) + elif n: + parts.append( + f" Of the excluded outputs that stay excluded, {words(n)} moved; each" + " was re-reviewed and keeps the value it was decided on" + " (excluded_outputs_rechecked)." + ) + else: + parts.append( + " No excluded output that stays excluded moved; each keeps the value it" + " was decided on." + ) + return "".join(parts) + + +def upgrade_exclusions(release: dict, plan: Plan, actions: dict) -> dict: + """The release record less the regenerated, the new records before the + trailing audit record, and the derivation naming the upgrade.""" + doc = copy.deepcopy(release) + gone = {key_of(r) for r in plan.regenerated} + records = [r for r in doc["exclusions"] if key_of(r) not in gone] + tail = records[-1:] if records and key_of(records[-1]) == AUDIT_TAIL else [] + body = records[: len(records) - len(tail)] + doc["exclusions"] = body + copy.deepcopy(plan.new_records) + tail + doc["derivation"] = doc["derivation"] + upgrade_sentence(plan, actions) + return doc + + +def upgrade_revision( + plan: Plan, + actions: dict, + *, + bundle: dict, + fix_modules: list[dict], + provenance: dict, +) -> dict: + engine = actions["engine"] + return { + "date": actions["date"], + "kind": "engine_upgrade", + "root_cause": "engine_upgrade_policyengine_us_" + + engine_number(engine).replace(".", "_"), + "outputs": "every scored output", + "rule": f"{RULE} {ENGINE_RULE} {UPGRADE_RULE}", + "engine_version": engine, + "previous_engine_version": actions["previous_engine"], + "policyengine_py": ( + f"policyengine {bundle.get('policyengine_version')}, provenance only: its " + "certified US bundle is policyengine-us " + f"{bundle.get('bundled_model_version')}" + ), + "fix_modules": fix_modules, + "builder": ( + "reference_audit/2026-10-12-engine-upgrade/scripts/" + "build_references_upgrade.py; households from " + "policybench.scenarios.Scenario.to_pe_household through " + "reference_audit/2026-09-28/scripts/sweep.py build_situation, as in the " + "2026-09-29 upgrade" + ), + "provenance": provenance, + "excluded_outputs_untouched": True, + "kept_exclusions_from_release": [ + {"scenario_id": k[0], "variable": k[1]} for k in plan.kept + ], + "excluded_outputs_rechecked": plan.rechecked, + "regenerated_exclusions": plan.regenerated, + "new_exclusions": [ + { + "scenario_id": r["scenario_id"], + "variable": r["variable"], + "reason_code": r["reason_code"], + "frozen_value": r["frozen_value"], + "alternative_value": r["alternative_value"], + "decided_on": r["decided_on"], + } + for r in plan.new_records + ], + "changed": plan.changed + plan.within, + } + + +def upgrade_meta( + base_meta: dict, revision: dict, *, csv_sha256: str, bundle: dict, now: str +) -> dict: + meta = copy.deepcopy(base_meta) + meta["policyengine_bundles"]["us"] = bundle + meta["reference_csv_sha256"] = csv_sha256 + meta["regenerated_at_utc"] = now + meta["revisions"].append(revision) + return meta + + +def verify_outputs( + base: Base, + plan: Plan, + csv_text: str, + exclusions: dict, + meta: dict, +) -> None: + """The written records agree: the loader accepts the exclusion record, every + excluded output carries its frozen value, every unchanged row keeps its + bytes, and every earlier revision is kept as it was.""" + import tempfile + + import pandas as pd + + from policybench.reference_exclusions import ( + load_reference_exclusions, + verify_exclusions_against_reference, + ) + + def check(condition: bool, message: str) -> None: + if not condition: + raise Refusal(f"written records disagree: {message}") + + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "reference_exclusions.json" + path.write_text(dump_record(exclusions)) + loaded = load_reference_exclusions(path) + verify_exclusions_against_reference(pd.read_csv(io.StringIO(csv_text)), loaded) + written = parse_reference_csv(csv_text) + check( + [r.key for r in written.rows] == [r.key for r in base.reference.rows], + "row order", + ) + for old, new in zip(base.reference.rows, written.rows): + if old.key in plan.new_values: + check(new.value == plan.new_values[old.key], f"{old.key} value") + else: + check(new.raw == old.raw, f"{old.key} bytes") + check(meta["revisions"][:-1] == base.meta["revisions"], "earlier revisions") + check( + dump_record({"r": meta["revisions"][:-1]}) + == dump_record({"r": base.meta["revisions"]}), + "earlier revision bytes", + ) + + +@dataclass +class Written: + csv_text: str + exclusions: dict + meta: dict + traces: dict + + +def write_outputs( + out_dir: Path, + base: Base, + release: dict, + plan: Plan, + actions: dict, + *, + bundle: dict, + fix_modules: list[dict], + provenance: dict, + traces: dict, + now: str, +) -> Written: + if now[:10] != actions["date"]: + raise Refusal( + f"regenerated_at_utc {now} does not fall on the actions date " + f"{actions['date']}; pass --regenerated-at or redate the actions" + ) + csv_text = render_reference_csv(base.reference, plan.new_values) + exclusions = upgrade_exclusions(release, plan, actions) + revision = upgrade_revision( + plan, actions, bundle=bundle, fix_modules=fix_modules, provenance=provenance + ) + meta = upgrade_meta( + base.meta, + revision, + csv_sha256=sha256_bytes(csv_text.encode()), + bundle=bundle, + now=now, + ) + verify_outputs(base, plan, csv_text, exclusions, meta) + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + (out / "reference_outputs.csv").write_text(csv_text) + (out / "reference_exclusions.json").write_text(dump_record(exclusions)) + (out / "reference_outputs.csv.meta.json").write_text(dump_record(meta)) + (out / "reference_traces.json").write_text(json.dumps(traces, indent=1)) + return Written(csv_text, exclusions, meta, traces) + + +def write_computed(path: Path, base: Base, computed: dict[Key, float], states) -> None: + """Every output on the new engine, in sweep.py's columns.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as sink: + writer = csv.writer(sink, lineterminator="\n") + writer.writerow( + [ + "fix", + "scenario_id", + "state", + "variable", + "frozen", + "recomputed", + "delta", + "moved", + ] + ) + for row in base.reference.rows: + if row.key not in computed: + continue + value = float(computed[row.key]) + writer.writerow( + [ + FIX_ENTRY, + row.key[0], + states.get(row.key[0], ""), + row.key[1], + repr(row.value), + repr(value), + repr(value - row.value), + beyond(row.key[1], row.value, value), + ] + ) + + +# --- Engine ------------------------------------------------------------------ + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def fix_module_pins(base_meta: dict, fixes_dir: Path) -> list[dict]: + """The modules latest_final.py composes, each the bytes the 2026-09-29 + revision pins, plus latest_final.py and the sales tax tables: the list the + 2026-10-10 revision pins, which must be exactly that.""" + upgrades = [r for r in base_meta["revisions"] if r["kind"] == "engine_upgrade"] + first = upgrades[0] + expected = [e["module"] for e in first["fix_modules"]] + expected += [f"{FIX_ENTRY}.py", SALES_TAX_TABLES] + if [e["module"] for e in upgrades[-1].get("fix_modules", [])] != expected: + raise Refusal( + "the base's last engine_upgrade revision does not pin the 2026-09-29 " + "modules plus latest_final.py and the sales tax tables" + ) + entries = [] + for entry in first["fix_modules"]: + path = fixes_dir / entry["module"] + if not path.is_file() or sha256(path) != entry["sha256"]: + raise Refusal(f"{path} is not the module the 2026-09-29 upgrade pins") + entries.append( + { + "module": entry["module"], + "path": f"reference_audit/2026-09-28/fixes/{entry['module']}", + "sha256": entry["sha256"], + } + ) + # Pinned to the bytes committed at the base commit, not to FIXES: imported + # in place, fixes_dir is FIXES and that comparison would be with itself. + entry_path = fixes_dir / f"{FIX_ENTRY}.py" + committed = git_blob(BASE_COMMIT, f"{FIXES_REL}/{FIX_ENTRY}.py") + if not entry_path.is_file() or sha256(entry_path) != sha256_bytes(committed): + raise Refusal( + f"{entry_path} is not {FIX_ENTRY}.py as committed at {BASE_COMMIT[:12]}" + ) + entries.append( + { + "module": f"{FIX_ENTRY}.py", + "path": f"reference_audit/2026-09-28/fixes/{FIX_ENTRY}.py", + "sha256": sha256(entry_path), + } + ) + tables = fixes_dir / SALES_TAX_TABLES + if not tables.is_file(): + raise Refusal( + f"{tables} is missing: latest_c_irs_sales_tax_2025.py reads it from its " + f"own directory; copy {SALES_TAX_SOURCE.relative_to(ROOT)} beside the " + "fixes" + ) + committed_tables = git_blob(BASE_COMMIT, SALES_TAX_REL) + if sha256(tables) != sha256_bytes(committed_tables) or sha256( + SALES_TAX_SOURCE + ) != sha256_bytes(committed_tables): + raise Refusal(f"{tables} is not {SALES_TAX_REL} as committed") + entries.append( + { + "module": SALES_TAX_TABLES, + "path": f"reference_audit/2026-09-22/fixes/{SALES_TAX_TABLES}", + "sha256": sha256(tables), + } + ) + return entries + + +def harness_pin(path: Path = HARNESS) -> dict: + """sweep.py, whose build_situation builds every household, must be the + bytes committed at the base commit.""" + committed = git_blob(BASE_COMMIT, HARNESS_REL) + if not Path(path).is_file() or sha256(path) != sha256_bytes(committed): + raise Refusal(f"{path} is not {HARNESS_REL} as committed at {BASE_COMMIT[:12]}") + return {"path": HARNESS_REL, "sha256": sha256_bytes(committed)} + + +def compute_outputs(system, scenarios, programs, build_situation) -> dict[Key, float]: + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + from policybench.spec import expand_programs_for_scenario + + computed = {} + for _, srow in scenarios.iterrows(): + scenario = scenario_from_dict(json.loads(srow["scenario_json"])) + situation = build_situation(scenario) + if scenario.id == "scenario_066": + head = next(iter(situation["people"].values())) + assert "weekly_hours_worked_before_lsr" in head, "builder hours mapping" + sim = Simulation(tax_benefit_system=system, situation=situation) + for variable in expand_programs_for_scenario(programs, scenario): + pe_variable = _pe_variable_for_output(variable, "us") + computed[(scenario.id, variable)] = float( + _extract_person_value( + sim.calculate(pe_variable, YEAR), scenario, variable + ) + ) + return computed + + +def module_dependencies(name: str, read) -> list[str]: + """The sibling files a fix module loads, transitively, in discovery order; + ``read(file)`` gives a file's bytes (as committed at BASE_COMMIT). Refuses + a module whose closure cannot be established + (policybench.fix_module_closure).""" + from policybench.fix_module_closure import ClosureError, dependency_closure + + try: + return dependency_closure(name, read) + except ClosureError as error: + raise Refusal(str(error)) from None + + +def fix_location(name: str) -> tuple[str, bytes]: + """An audited fix file's repository path and committed bytes: under + AUDIT_FIXES_REL as committed at BASE_COMMIT, or, for a module this audit + adds (absent there), under LOCAL_FIXES_REL as committed at HEAD.""" + try: + return f"{AUDIT_FIXES_REL}/{name}", git_blob(BASE_COMMIT, f"{AUDIT_FIXES_REL}/{name}") + except Refusal: + pass + try: + return f"{LOCAL_FIXES_REL}/{name}", git_blob("HEAD", f"{LOCAL_FIXES_REL}/{name}") + except Refusal: + raise Refusal( + f"{name} is neither in {AUDIT_FIXES_REL} at {BASE_COMMIT[:12]} nor " + f"committed at HEAD in {LOCAL_FIXES_REL}" + ) from None + + +def is_local_fix(name: str) -> bool: + """Whether a fix module is this audit's own (fix_location); a name found + in neither place is not.""" + try: + return fix_location(name)[0].startswith(LOCAL_FIXES_REL + "/") + except Refusal: + return False + + +def committed_fix(name: str) -> bytes: + """An audited fix file's committed bytes (fix_location).""" + return fix_location(name)[1] + + +def fix_path(name: str, fixes_dir: Path = AUDIT_FIXES) -> Path: + """Where a fix module is read from: ``fixes_dir`` for the 2026-09-22 + audit's modules, LOCAL_FIXES for this audit's own.""" + return (LOCAL_FIXES if is_local_fix(name) else Path(fixes_dir)) / name + + +def audit_module_pins(names, fixes_dir: Path = AUDIT_FIXES) -> dict[str, str]: + """Each named audited fix module's sha256, and each sibling it loads + (module_dependencies): the bytes committed at BASE_COMMIT under + reference_audit/2026-09-22/fixes, or, for a module this audit adds, the + bytes committed at HEAD under LOCAL_FIXES_REL (fix_location); and nothing + else.""" + names = sorted(set(names)) + for name in list(names): + names += [d for d in module_dependencies(name, committed_fix) if d not in names] + pins = {} + for name in sorted(set(names)): + if Path(name).name != name or not name.endswith((".py", ".json")): + raise Refusal(f"a fix module is a file name ending in .py, not {name!r}") + rel, blob = fix_location(name) + path = fix_path(name, fixes_dir) + committed = sha256_bytes(blob) + if not path.is_file() or sha256(path) != committed: + where = BASE_COMMIT[:12] if rel.startswith(AUDIT_FIXES_REL) else "HEAD" + raise Refusal(f"{path} is not {rel} as committed at {where}") + pins[name] = committed + return pins + + +def fix_module_parts(names, fixes_dir: Path = AUDIT_FIXES) -> tuple[list, list]: + """The Reform classes and household patches of the named fix modules, in + order (sweep.py's fix interface: a module defines reform, patch or both).""" + reforms, patches = [], [] + for name in names: + module = load_module(f"upgrade_fix_{Path(name).stem}", fix_path(name, fixes_dir)) + reform = getattr(module, "reform", None) + patch = getattr(module, "patch", None) + if reform is None and patch is None: + raise Refusal(f"{name} defines neither reform nor patch") + if reform is not None: + reforms.append(reform) + if patch is not None: + patches.append(patch) + return reforms, patches + + +def corrected_system(final, reforms): + """The pinned conventions, then each fix module's reform, as one system.""" + from policyengine_core.reforms import Reform + from policyengine_us import CountryTaxBenefitSystem + + parts = (final.reform, *reforms) + + class corrected(Reform): + def apply(self): + for part in parts: + part.apply(self) + + return CountryTaxBenefitSystem(reform=corrected) + + +def compute_items(system, scenarios, keys, build_situation, patches=()) -> dict: + """The named outputs on a system, households built as compute_outputs + builds them and then edited by each patch in order.""" + from policyengine_us import Simulation + + from policybench.ground_truth import _extract_person_value, _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + + by_id = scenarios.set_index("scenario_id") + out = {} + for scenario_id in sorted({k[0] for k in keys}): + scenario = scenario_from_dict( + json.loads(by_id.loc[scenario_id, "scenario_json"]) + ) + situation = build_situation(scenario) + for patch in patches: + situation = patch(copy.deepcopy(situation), scenario) + sim = Simulation(tax_benefit_system=system, situation=situation) + for k in sorted(k for k in keys if k[0] == scenario_id): + pe_variable = _pe_variable_for_output(k[1], "us") + out[k] = float( + _extract_person_value(sim.calculate(pe_variable, YEAR), scenario, k[1]) + ) + return out + + +def module_values_on(final, scenarios, requests, build_situation) -> dict: + """Each (output, modules) request's value with the conventions and the + modules applied, one system per module list.""" + values = {} + by_modules: dict[tuple[str, ...], list[Key]] = {} + for k, modules in requests: + by_modules.setdefault(tuple(modules), []).append(k) + for modules, keys in sorted(by_modules.items()): + reforms, patches = fix_module_parts(modules) + system = corrected_system(final, reforms) + for k, value in compute_items( + system, scenarios, keys, build_situation, patches + ).items(): + values[(k, modules)] = value + return values + + +def load_evidence(ref: dict) -> dict: + """An evidence file a target names, at its pinned sha256.""" + if not isinstance(ref, dict) or set(ref) != {"path", "sha256"}: + raise Refusal(f"a target's evidence is {{path, sha256}}, not {ref!r}") + path = ROOT / ref["path"] + if not path.is_file() or sha256(path) != ref["sha256"]: + raise Refusal(f"{ref['path']} is missing or not the evidence its sha256 pins") + doc = json.loads(path.read_text()) + if not isinstance(doc, dict) or doc.get("kind") != EVIDENCE_KIND: + raise Refusal(f"{ref['path']} is not {EVIDENCE_KIND}") + engine_number(doc.get("engine", "")) + if not isinstance(doc.get("modules"), dict) or not isinstance( + doc.get("items"), list + ): + raise Refusal(f"{ref['path']} lacks its modules or items") + return doc + + +def write_evidence(request_path: Path, out_path: Path, fixes_dir: Path) -> dict: + """Evidence mode: each requested output on the installed engine, with the + pinned conventions alone and with its fix modules after them.""" + from importlib.metadata import version + + import pandas as pd + + request = json.loads(Path(request_path).read_text()) + items = request.get("items") if isinstance(request, dict) else None + if not isinstance(items, list) or not items: + raise Refusal("an evidence request is {\"items\": [...]} with one or more") + base = load_base() + board = base.reference.values() + requests = [] + for item in items: + k, modules = key_of(item), tuple(item.get("modules") or ()) + if k not in board or not modules: + raise Refusal(f"evidence request {k}: not a reference, or no modules") + requests.append((k, modules)) + if len(set(requests)) != len(requests): + raise Refusal("an evidence request names an item twice") + pins = audit_module_pins([m for _, modules in requests for m in modules]) + fix_module_pins(base.meta, fixes_dir) + harness_pinned = harness_pin() + harness = load_module("upgrade_sweep_harness", HARNESS) + final = load_module(f"upgrade_{FIX_ENTRY}", fixes_dir / f"{FIX_ENTRY}.py") + + from policyengine_us import CountryTaxBenefitSystem + + scenarios = pd.read_csv(io.StringIO(base.scenarios_csv)) + plain = compute_items( + CountryTaxBenefitSystem(reform=final.reform), + scenarios, + [k for k, _ in requests], + harness.build_situation, + ) + corrected = module_values_on(final, scenarios, requests, harness.build_situation) + doc = { + "kind": EVIDENCE_KIND, + "engine": f"policyengine-us {version('policyengine-us')}", + "policyengine_core": version("policyengine-core"), + "computed_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "builder_sha256": sha256(Path(__file__)), + "harness_sha256": harness_pinned["sha256"], + "conventions": f"{FIXES_REL}/{FIX_ENTRY}.py", + "base_commit": BASE_COMMIT, + "modules": pins, + "items": [ + { + "scenario_id": k[0], + "variable": k[1], + "modules": list(modules), + "engine_value": plain[k], + "corrected_value": corrected[(k, modules)], + } + for k, modules in requests + ], + } + out = Path(out_path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(dump_record(doc)) + return doc + + +def trace_outputs(system, scenarios, items, build_situation) -> dict: + import types + + sys.modules.setdefault("litellm", types.ModuleType("litellm")) + from policyengine_us import Simulation + + from policybench.case_reference_explanations import _find_target_tree, _render_trace + from policybench.ground_truth import _pe_variable_for_output + from policybench.scenarios import scenario_from_dict + + by_id = scenarios.set_index("scenario_id") + traces = {} + for item in items: + scenario = scenario_from_dict( + json.loads(by_id.loc[item["scenario_id"], "scenario_json"]) + ) + sim = Simulation(tax_benefit_system=system, situation=build_situation(scenario)) + sim.trace = True + pe_variable = _pe_variable_for_output(item["variable"], "us") + sim.calculate(pe_variable, YEAR) + tree = _find_target_tree(sim.tracer.trees, pe_variable) + traces[f"{item['scenario_id']}|{item['variable']}"] = { + "pe_variable": pe_variable, + "trace": "\n".join(_render_trace(tree)) if tree else "", + } + return traces + + +def main(argv: list[str] | None = None) -> None: + from importlib.metadata import version + + import pandas as pd + + parser = argparse.ArgumentParser() + parser.add_argument("--actions") + parser.add_argument("--out-dir") + parser.add_argument("--fixes-dir", default=str(FIXES)) + parser.add_argument("--computed-csv") + parser.add_argument("--regenerated-at") + parser.add_argument("--allow-draft", action="store_true") + parser.add_argument("--evidence-request") + parser.add_argument("--evidence-out") + args = parser.parse_args(argv) + + if args.evidence_request or args.evidence_out: + if not (args.evidence_request and args.evidence_out) or args.actions: + parser.error("evidence mode takes --evidence-request and --evidence-out") + doc = write_evidence( + Path(args.evidence_request), + Path(args.evidence_out), + Path(args.fixes_dir).resolve(), + ) + print(f"{doc['engine']}: {len(doc['items'])} items") + for item in doc["items"]: + print( + f" {item['scenario_id']} {item['variable']:46s} " + f"{item['engine_value']:>11.2f} -> {item['corrected_value']:>11.2f} " + f"{'+'.join(item['modules'])}" + ) + return + if not (args.actions and args.out_dir): + parser.error("a build takes --actions and --out-dir") + + actions_path = Path(args.actions) + actions = json.loads(actions_path.read_text()) + validate_actions(actions, allow_draft=args.allow_draft) + engine = engine_number(actions["engine"]) + installed = version("policyengine-us") + if installed != engine: + raise Refusal(f"need policyengine-us {engine}, have {installed}") + + base = load_base() + release, ruled, spec_sha = release_record(base.exclusions) + if actions.get("release_spec_sha256") != spec_sha: + drafted = actions.get("release_spec_sha256") + raise Refusal( + f"the actions were drafted against spec {drafted}, not " + f"docs/musewonder/spec.json {spec_sha}" + ) + fixes_dir = Path(args.fixes_dir).resolve() + fix_modules = fix_module_pins(base.meta, fixes_dir) + harness_pinned = harness_pin() + harness = load_module("upgrade_sweep_harness", HARNESS) + final = load_module(f"upgrade_{FIX_ENTRY}", fixes_dir / f"{FIX_ENTRY}.py") + + from policyengine_us import CountryTaxBenefitSystem + + from policybench.policyengine_runtime import policyengine_release_bundle + + system = CountryTaxBenefitSystem(reform=final.reform) + scenarios = pd.read_csv(io.StringIO(base.scenarios_csv)) + computed = compute_outputs( + system, scenarios, base.meta["programs"], harness.build_situation + ) + states = dict(zip(scenarios["scenario_id"], scenarios["state"])) + out = Path(args.out_dir) + write_computed( + Path(args.computed_csv or out / "computed.csv"), base, computed, states + ) + targets = [ + entry + for entry in actions.get("regenerated_exclusions", []) + if isinstance(entry.get("target"), dict) + and entry["target"].get("kind") == TARGET_FIX_MODULES + ] + evidence = { + entry["target"]["evidence"]["path"]: load_evidence(entry["target"]["evidence"]) + for entry in targets + if isinstance(entry["target"].get("evidence"), dict) + } + named = sorted({m for entry in targets for m in target_modules(entry)}) + module_pins = audit_module_pins(named) + module_deps = {m: module_dependencies(m, committed_fix) for m in named} + module_values = module_values_on( + final, + scenarios, + [(key_of(entry), target_modules(entry)) for entry in targets], + harness.build_situation, + ) + plan = plan_upgrade( + base, + release, + ruled, + actions, + computed, + evidence=evidence, + module_pins=module_pins, + module_values=module_values, + module_deps=module_deps, + ) + if plan.problems: + raise Refusal("refusing to write references:\n " + "\n ".join(plan.problems)) + + traces = trace_outputs( + system, scenarios, plan.changed + plan.within, harness.build_situation + ) + now = ( + args.regenerated_at or datetime.datetime.now(datetime.timezone.utc).isoformat() + ) + provenance = { + "builder_sha256": sha256(Path(__file__)), + "harness": harness_pinned["path"], + "harness_sha256": harness_pinned["sha256"], + "actions_sha256": sha256(actions_path), + "release_spec_sha256": spec_sha, + "base_commit": BASE_COMMIT, + "policyengine_core": version("policyengine-core"), + } + written = write_outputs( + out, + base, + release, + plan, + actions, + bundle=policyengine_release_bundle("us"), + fix_modules=fix_modules, + provenance=provenance, + traces=traces, + now=now, + ) + print(f"sha256 {written.meta['reference_csv_sha256']}") + print( + f"{len(plan.changed)} reviewed changes, {len(plan.within)} within $1, " + f"{len(plan.regenerated)} regenerated exclusions, {len(plan.new_records)} new " + f"exclusions, {len(plan.rechecked)} excluded outputs rechecked, " + f"{len(written.exclusions['exclusions'])} exclusion records" + ) + for c in plan.changed + plan.within: + print( + f" {c['scenario_id']} {c['variable']:46s} {c['previous']:>11.2f} -> " + f"{c['regenerated']:>11.2f} {c['cause']}" + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-10-12-engine-upgrade/scripts/check_against_evidence.py b/reference_audit/2026-10-12-engine-upgrade/scripts/check_against_evidence.py new file mode 100644 index 00000000..c789bb75 --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/scripts/check_against_evidence.py @@ -0,0 +1,111 @@ +"""Check an engine against the fix modules' pre-fix evidence, before it ships. + +For each evidence item (reference_audit/2026-10-12-engine-upgrade/evidence/), +compute the output on the importable policyengine-us, with the pinned +conventions alone and with the item's fix modules after them, and report: + + lands the engine is within $1 of the evidence's corrected value (a flag + equals it), which a regeneration needs; + inert the modules move it by no more than $1 there: nothing is left to fix. + +This is the builder's regeneration gate (build_references_upgrade.py, +regeneration_target) run early, so a policyengine-us pull request can be checked +before it merges. PYTHONPATH may put a policyengine-us checkout ahead of the +installed one; the engine is then labelled by --label, since the installed +package's version would misname it. Nothing is written; the exit code is 1 when +any checked item fails. + + PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 \\ + PYTHONPATH=: \\ + /bin/python .../check_against_evidence.py \\ + --evidence reference_audit/2026-10-12-engine-upgrade/evidence/pe2.37.2.json \\ + --label "PR #10032 head abc1234" [--module r02_ira_219g_v2.py ...] +""" + +from __future__ import annotations + +import argparse +import io +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import build_references_upgrade as build # noqa: E402 + +TOL = 1.0 + + +def main(argv: list[str] | None = None) -> int: + import pandas as pd + import policyengine_us + from policyengine_us import CountryTaxBenefitSystem + + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", required=True) + parser.add_argument("--label", required=True) + parser.add_argument( + "--module", + action="append", + default=[], + help="check only items whose modules include this one (repeatable)", + ) + parser.add_argument("--fixes-dir", default=str(build.FIXES)) + args = parser.parse_args(argv) + + path = build.ROOT / args.evidence + doc = build.load_evidence({"path": args.evidence, "sha256": build.sha256(path)}) + items = [ + item + for item in doc["items"] + if not args.module or set(args.module) & set(item["modules"]) + ] + if not items: + raise build.Refusal("no evidence item uses the named modules") + pins = build.audit_module_pins([m for item in items for m in item["modules"]]) + stale = sorted(m for m, pin in pins.items() if doc["modules"].get(m) != pin) + if stale: + raise build.Refusal(f"the evidence ran other bytes of {stale}") + base = build.load_base() + fixes_dir = Path(args.fixes_dir).resolve() + build.fix_module_pins(base.meta, fixes_dir) + build.harness_pin() + harness = build.load_module("upgrade_sweep_harness", build.HARNESS) + final = build.load_module(f"upgrade_{build.FIX_ENTRY}", fixes_dir / "latest_final.py") + scenarios = pd.read_csv(io.StringIO(base.scenarios_csv)) + requests = [(build.key_of(item), tuple(item["modules"])) for item in items] + plain = build.compute_items( + CountryTaxBenefitSystem(reform=final.reform), + scenarios, + [k for k, _ in requests], + harness.build_situation, + ) + corrected = build.module_values_on( + final, scenarios, requests, harness.build_situation + ) + print( + f"{args.label} (policyengine_us from {Path(policyengine_us.__file__).parent}) " + f"against {doc['engine']} evidence" + ) + failed = 0 + for item, (k, modules) in zip(items, requests): + value, after = plain[k], corrected[(k, modules)] + target = float(item["corrected_value"]) + lands = abs(value - target) <= TOL and not build.beyond(k[1], target, value) + inert = abs(after - value) <= TOL and not build.beyond(k[1], value, after) + failed += not (lands and inert) + print( + f" {'PASS' if lands and inert else 'FAIL'} {k[0]} {k[1]:44s} " + f"engine {value:>11.2f} target {target:>11.2f} " + f"{'lands' if lands else 'MISSES by ' + format(value - target, '+.2f')}; " + f"{'inert' if inert else 'modules still move it to ' + format(after, '.2f')}" + f" [{'+'.join(modules)}]" + ) + print(f"{len(items) - failed} of {len(items)} pass") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/reference_audit/2026-10-12-engine-upgrade/scripts/narratives_upgrade.py b/reference_audit/2026-10-12-engine-upgrade/scripts/narratives_upgrade.py new file mode 100644 index 00000000..3b8647da --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/scripts/narratives_upgrade.py @@ -0,0 +1,429 @@ +"""Rewrite the reference explanations of the outputs the third engine upgrade +changed; every other row of the explanations CSV keeps its bytes. + +Adapted from reference_audit/2026-09-28/scripts/narratives_latest.py. The writer +is every reference narrative's (policybench.case_reference_explanations: +REFERENCE_MODEL claude-haiku-4-5, TEMPERATURE, MAX_TOKENS, _prompt), given the +build's engine trace (reference_traces.json from build_references_upgrade.py) and +the change's recorded basis as grounding. A new exclusion is grounded on its +record's alternative reading, as in 2026-09-28. The narrative must state the new +reference value (an amount; a flag states none); a draft that omits it is retried +twice, then refused. --hand-corrected takes a JSON object +{"scenario_id|variable": text} that replaces the writer for a listed output; its +text must also state the value. + +--reuse-from --reuse-explanations keeps +an earlier build's narrative for an output whose writer inputs are the same in +both builds: the value, the change's cause and grounding, the PolicyEngine +variable and the trace, byte for byte. On another engine the narrative must +also not name the earlier engine's version. Judges' prompts render the narrative, so a reused one +keeps a case's verdict through a rebuild on a newer engine. The reused rows are +listed beside --out (.reused.json) with both builds' sidecar sha256. + +The rows rewritten are exactly the sidecar's last revision's changed list (a +reference that changed by more than EPS); each gets the new reference_value, +trace_lines and explanation, and an empty error. Judge prompts render a case's +reference value and explanation, so these are the cases whose verdicts +prepare_audit drops and the judge re-runs. + +The writer is Anthropic's, through litellm: the run needs ANTHROPIC_API_KEY and +never an OpenAI key (the script removes OPENAI_API_KEY from its environment): + + ANTHROPIC_API_KEY=$(agent-secret get ANTHROPIC_API_KEY maxghenis) \\ + PYTHONPATH= /bin/python narratives_upgrade.py \\ + --references \\ + --explanations annotations//us_case_reference_explanations.csv \\ + --out +""" + +from __future__ import annotations + +import argparse +import csv +import io +import json +import math +import os +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from build_references_upgrade import ( # noqa: E402 + BASE_COMMIT, + BASE_SHA256, + RUN, + YEAR, + Refusal, + format_row, + git_blob, + physical_lines, + sha256_bytes, +) + +FIELDS = [ + "country", + "scenario_id", + "variable", + "reference_value", + "trace_lines", + "explanation", + "error", +] +RETRIES = 2 + + +def money(value: float) -> str: + return f"{value:,.2f}".removesuffix(".00") + + +# A dollar amount as a narrative writes it: "$1,916.61", "$0", "-$12.50". +DOLLARS = re.compile(r"(-)?\$\s?(\d[\d,]*(?:\.\d+)?)") + + +def required(item: dict) -> list[float]: + """What the narrative must state: the new amount; nothing for a flag.""" + if item["variable"].endswith("_eligible"): + return [] + return [float(item["regenerated"])] + + +def states_amount(text: str, value: float) -> bool: + """Whether ``text`` states ``value`` as a dollar amount, to the cent. A + bare number does not count: for a $0 reference, "2026" states nothing.""" + for sign, number in DOLLARS.findall(text): + amount = float(number.replace(",", "")) * (-1 if sign else 1) + if abs(amount - value) <= 0.005 + 1e-9: + return True + return False + + +def missing_amounts(item: dict, text: str) -> list[str]: + return [money(v) for v in required(item) if not states_amount(text, v)] + + +def engine_note(engine: str) -> str: + return ( + f" PolicyEngine here is {engine} with the benchmark's conventions for law " + "published before July 3, 2026." + ) + + +def grounding_for(item: dict, records: dict) -> str: + """The change's recorded basis; a new exclusion's record reading instead.""" + record = records.get((item["scenario_id"], item["variable"])) + if record is not None and item["cause"] == f"excluded_{record['reason_code']}": + if record["reason_code"] == "reference_depends_on_unlisted_input": + return ( + "PolicyBench does not score this output, because the reference " + "depends on an input the prompt does not state. " + + record["alternative_reading"] + ) + return ( + "PolicyBench does not score this output. " + record["alternative_reading"] + ) + return item["basis"] + + +def parse_rows(text: str) -> tuple[str, list[tuple[list[str], str]]]: + """The header line and each record with its raw text.""" + lines = physical_lines(text) + reader = csv.reader(iter(lines)) + records, used = [], 0 + for fields in reader: + records.append((fields, "".join(lines[used : reader.line_num]))) + used = reader.line_num + (header, header_raw), body = records[0], records[1:] + if header != FIELDS: + raise Refusal(f"unexpected explanations header {header}") + return header_raw, body + + +def rewrite_explanations(text: str, rewrites: dict) -> str: + """``text`` with each listed (scenario_id, variable) row replaced by + (reference_value, trace_lines, explanation); every other row byte for byte.""" + header_raw, body = parse_rows(text) + keys = [(fields[1], fields[2]) for fields, _ in body] + for key in rewrites: + if keys.count(key) != 1: + raise Refusal(f"{key} is in the explanations {keys.count(key)} times") + parts = [header_raw] + for fields, raw in body: + key = (fields[1], fields[2]) + if key not in rewrites: + parts.append(raw) + continue + value, n_lines, explanation = rewrites[key] + row = format_row( + [ + fields[0], + key[0], + key[1], + repr(float(value)), + str(n_lines), + explanation, + "", + ] + ) + parts.append(row if raw.endswith("\n") else row.removesuffix("\n")) + out = "".join(parts) + # Every row the rewrite does not name keeps its bytes. + _, written = parse_rows(out) + for (old, old_raw), (new, new_raw) in zip(body, written, strict=True): + if (old[1], old[2]) not in rewrites and old_raw != new_raw: + raise Refusal(f"row {(old[1], old[2])} changed") + return out + + +def strip_heading(text: str) -> str: + text = text.strip() + lines = text.split("\n") + if lines and lines[0].lstrip().startswith("#"): + text = "\n".join(lines[1:]).strip() + return text + + +def write_narratives( + references: Path, + explanations_text: str, + scenarios, + *, + completion, + hand_corrected: dict | None = None, + reuse: dict | None = None, +) -> tuple[str, list[tuple]]: + """The explanations CSV with every changed output's narrative rewritten, + and the rewritten rows. ``completion`` is litellm.completion (mocked in + tests). ``reuse`` maps an output to the earlier narrative reusable_narratives + found for it; the writer is not called for those.""" + from policybench.case_reference_explanations import ( + MAX_TOKENS, + REFERENCE_MODEL, + TEMPERATURE, + _prompt, + _scenario_summary, + ) + + if not REFERENCE_MODEL.startswith("claude-"): + raise Refusal( + f"the narrative writer must be Anthropic's, not {REFERENCE_MODEL}" + ) + meta = json.loads((references / "reference_outputs.csv.meta.json").read_text()) + upgrade = meta["revisions"][-1] + if ( + upgrade["kind"] != "engine_upgrade" + or len([r for r in meta["revisions"] if r["kind"] == "engine_upgrade"]) < 3 + ): + raise Refusal("the sidecar's last revision is not the third engine upgrade") + traces = json.loads((references / "reference_traces.json").read_text()) + exclusions = json.loads((references / "reference_exclusions.json").read_text()) + records = {(e["scenario_id"], e["variable"]): e for e in exclusions["exclusions"]} + note = engine_note(upgrade["engine_version"]) + hand_corrected = hand_corrected or {} + reuse = reuse or {} + by_id = scenarios.set_index("scenario_id", drop=False) + + def write(item: dict, extra: str = "") -> str: + key = (item["scenario_id"], item["variable"]) + traced = traces[f"{key[0]}|{key[1]}"] + prompt = _prompt( + "us", + _scenario_summary(by_id.loc[key[0]]), + key[1], + traced["pe_variable"], + item["regenerated"], + YEAR, + traced["trace"], + grounding=grounding_for(item, records) + note + extra, + ) + response = completion( + model=REFERENCE_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, + ) + return strip_heading(response.choices[0].message.content) + + rewrites, results = {}, [] + for item in upgrade["changed"]: + key = (item["scenario_id"], item["variable"]) + n_lines = len(traces[f"{key[0]}|{key[1]}"]["trace"].splitlines()) + listed = f"{key[0]}|{key[1]}" + if listed in hand_corrected: + text = hand_corrected[listed] + elif key in reuse: + text = reuse[key] + else: + text = write(item) + for _ in range(RETRIES): + missing = missing_amounts(item, text) + if not missing: + break + text = write( + item, + " The narrative must state the value " + + ", ".join(f"${m}" for m in missing) + + ".", + ) + missing = missing_amounts(item, text) + if missing: + raise Refusal(f"{key[0]} {key[1]}: the narrative omits {missing}") + rewrites[key] = (item["regenerated"], n_lines, text) + results.append((key[0], key[1], item["regenerated"], n_lines, text)) + unused = set(hand_corrected) - {f"{k[0]}|{k[1]}" for k in rewrites} + if unused: + raise Refusal(f"hand-corrected outputs the upgrade did not change: {unused}") + return rewrite_explanations(explanations_text, rewrites), results + + +def writer_inputs(references: Path) -> dict: + """Each changed output's narrative-writer inputs in a build, by output: + value, cause, grounding, PolicyEngine variable and trace.""" + meta = json.loads((references / "reference_outputs.csv.meta.json").read_text()) + traces = json.loads((references / "reference_traces.json").read_text()) + exclusions = json.loads((references / "reference_exclusions.json").read_text()) + records = {(e["scenario_id"], e["variable"]): e for e in exclusions["exclusions"]} + inputs = {} + for item in meta["revisions"][-1]["changed"]: + key = (item["scenario_id"], item["variable"]) + traced = traces[f"{key[0]}|{key[1]}"] + inputs[key] = ( + repr(float(item["regenerated"])), + item["cause"], + grounding_for(item, records), + traced["pe_variable"], + traced["trace"], + ) + return inputs + + +def reusable_narratives( + references: Path, earlier: Path, earlier_explanations_text: str +) -> dict: + """The earlier build's narratives for outputs whose writer inputs both + builds share and whose narrative does not name the earlier engine.""" + def engine(path: Path) -> str: + meta = json.loads((path / "reference_outputs.csv.meta.json").read_text()) + return meta["revisions"][-1]["engine_version"] + + now, then = engine(references), engine(earlier) + number = then.removeprefix("policyengine-us ") + current = writer_inputs(references) + # The groundings name their build's engine; that name may differ, nothing else. + before = { + key: (*inputs[:2], inputs[2].replace(then, now), *inputs[3:]) + for key, inputs in writer_inputs(earlier).items() + } + _, body = parse_rows(earlier_explanations_text) + rows: dict = {} + for fields, _ in body: + rows.setdefault((fields[1], fields[2]), []).append(fields) + # The CSV must be the earlier build's narratives: each output that build + # changed appears once, for the US, with no error, at that build's value. + # Each value must parse to a finite number: abs(nan - x) > 1e-6 is false, + # so a "nan" would pass a bare comparison. + wrong = sorted( + key + for key, inputs in writer_inputs(earlier).items() + if len(rows.get(key, [])) != 1 + or rows[key][0][0] != "us" + or rows[key][0][6] != "" + or (stated := finite(rows[key][0][3])) is None + or (value := finite(inputs[0])) is None + or abs(stated - value) > 1e-6 + ) + if wrong: + raise Refusal( + "--reuse-explanations is not --reuse-from's narratives: " + f"{[f'{k[0]}|{k[1]}' for k in wrong[:5]]}" + ) + narratives = {key: found[0][5] for key, found in rows.items()} + # A narrative naming the earlier engine is stale on another engine, and + # still true on the same one. + return { + key: narratives[key] + for key, inputs in current.items() + if before.get(key) == inputs and (then == now or number not in narratives[key]) + } + + +def finite(text) -> float | None: + """A value's text as a finite float, or None (unparsable, nan or inf).""" + try: + value = float(text) + except (TypeError, ValueError): + return None + return value if math.isfinite(value) else None + + +def main(argv: list[str] | None = None) -> None: + import pandas as pd + + parser = argparse.ArgumentParser() + parser.add_argument("--references", required=True) + parser.add_argument("--explanations", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--hand-corrected") + parser.add_argument("--reuse-from") + parser.add_argument("--reuse-explanations") + args = parser.parse_args(argv) + if bool(args.reuse_from) != bool(args.reuse_explanations): + parser.error("--reuse-from and --reuse-explanations go together") + os.environ.pop("OPENAI_API_KEY", None) + if not os.environ.get("ANTHROPIC_API_KEY"): + raise Refusal( + "set ANTHROPIC_API_KEY (agent-secret get ANTHROPIC_API_KEY maxghenis)" + ) + import litellm + + raw = git_blob(BASE_COMMIT, f"{RUN}/scenarios.csv") + if sha256_bytes(raw) != BASE_SHA256["scenarios.csv"]: + raise Refusal("scenarios.csv at the base commit does not match its pin") + scenarios = pd.read_csv(io.StringIO(raw.decode())) + hand = ( + json.loads(Path(args.hand_corrected).read_text()) if args.hand_corrected else {} + ) + reuse = {} + if args.reuse_from: + reuse = reusable_narratives( + Path(args.references), + Path(args.reuse_from), + Path(args.reuse_explanations).read_text(), + ) + reuse = {k: v for k, v in reuse.items() if f"{k[0]}|{k[1]}" not in hand} + text, results = write_narratives( + Path(args.references), + Path(args.explanations).read_text(), + scenarios, + completion=litellm.completion, + hand_corrected=hand, + reuse=reuse, + ) + Path(args.out).write_text(text) + if args.reuse_from: + sidecar = "reference_outputs.csv.meta.json" + Path(args.out + ".reused.json").write_text( + json.dumps( + { + "reused": [f"{k[0]}|{k[1]}" for k in sorted(reuse)], + "from_sidecar_sha256": sha256_bytes( + (Path(args.reuse_from) / sidecar).read_bytes() + ), + "into_sidecar_sha256": sha256_bytes( + (Path(args.references) / sidecar).read_bytes() + ), + }, + indent=2, + ) + + "\n" + ) + print(f"reused {len(reuse)} narratives from {args.reuse_from}") + for scenario_id, variable, value, _, narrative in results: + print(f"--- {scenario_id} {variable} ({value:,.2f})\n{narrative}\n") + print(f"rewrote {len(results)} narratives -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py b/reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py new file mode 100644 index 00000000..2d6f09b1 --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py @@ -0,0 +1,790 @@ +"""Record when the 2026-10-12 reference sweep began and check the newest +policyengine-us release before publishing, as reference_audit/2026-09-28 and +reference_audit/2026-10-09-engine-upgrade did for the 2026-09-29 and 2026-10-10 +moves (verification/sweep_timing.json in each). + +The standing rule (build_references_upgrade.py's ENGINE_RULE): references +come from the newest policyengine-us release when PolicyBench begins the +reference sweep, and at publication PolicyBench checks that the newest +release gives the same values. + +Two steps, each writing into this audit's verification/ directory: + + sweep_timing.py sweep --engine 2.39.0 --venv --first-output + Reads PyPI's JSON API now and records each release's wheel upload time + from the reference engine onward, the newest release at the read, when + the engine was installed (the dist-info directory's birth time) and + when the sweep wrote its first output (the file's birth time). Refuses + unless the engine was the newest release when the sweep began: no + release newer than it was uploaded before its first output. + + sweep_timing.py run --venv --engine X.Y.Z --receipt -- + Runs the builder for the reference build through the same engine check, + in the same single process, as the check sweep (ENGINE_RUNNER); the + receipt records the check. + + sweep_timing.py check --build + [--check-venv ] + Reads PyPI again. The build must be the release's: its sidecar names the + sweep's engine, pins its CSV and names the committed builder, its + reference CSV, sidecar and exclusion record are the committed snapshot's + byte for byte, and its computed.csv gives every scored reference, every + value finite. When a newer release is out, check runs the check sweep + itself: this audit's builder, a first pass with empty actions, in the + given venv, which the builder refuses unless it holds that release. It + compares every output that sweep computes with the reference engine's + (publication_check_.csv) and records the result, counting + scored and excluded outputs apart; when the reference engine is still + the newest, it records that. Each input is pinned in the record by + sha256. Exits non-zero, after writing the record, when the newest + release moves a scored output: that stops the publish. + +A release whose wheels are all yanked is not one PolicyBench builds on or +checks against; the record lists any such release newer than the reference +engine (yanked_newer), so one yanked after the sweep began stays visible. + +Times are UTC. Birth times are macOS st_birthtime. +""" + +from __future__ import annotations + +import argparse +import csv +import datetime +import hashlib +import json +import math +import os +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +AUDIT = Path(__file__).resolve().parents[1] +VERIFICATION = AUDIT / "verification" +TIMING = VERIFICATION / "sweep_timing.json" +PYPI = "https://pypi.org/pypi/policyengine-us/json" +PACKAGE = "policyengine_us" +SAME = 1e-9 +# The builder's own tolerance for an output it did not move (its EPS): every +# scored reference is the engine's value, so the build's computed.csv gives it. +REFERENCE_EPS = 1e-6 +RUN = "us_full_run_20260612_policyengine_4_16_1_populace" +SNAPSHOT = ROOT / "paper/snapshot/20260501/runs" / RUN +BUILDER = AUDIT / "scripts" / "build_references_upgrade.py" +SPEC = ROOT / "docs/musewonder/spec.json" +# The builder's arguments that name files or directories, and its flag. +BUILDER_PATH_FLAGS = ( + "--actions", + "--out-dir", + "--fixes-dir", + "--computed-csv", + "--evidence-request", + "--evidence-out", +) +BUILDER_VALUE_FLAGS = (*BUILDER_PATH_FLAGS, "--regenerated-at") +BUILDER_SWITCHES = ("--allow-draft",) + + +def builder_arguments(argv: list[str]) -> tuple[list[str], Path | None]: + """The builder's arguments as the builder itself reads them (the same + flags, argparse's equals form, abbreviations and last-occurrence rule), + rewritten in one canonical form with every path resolved here (the child + runs in the repository root), and the computed.csv the run must write: + --computed-csv when given, else /computed.csv. Unknown arguments + are refused rather than passed on.""" + parser = argparse.ArgumentParser(prog="builder", add_help=False) + for flag in BUILDER_VALUE_FLAGS: + parser.add_argument(flag) + for flag in BUILDER_SWITCHES: + parser.add_argument(flag, action="store_true") + try: + parsed = parser.parse_args(argv) + except SystemExit: + raise Refusal(f"the builder cannot take these arguments: {argv}") from None + canonical: list[str] = [] + values = vars(parsed) + for flag in BUILDER_VALUE_FLAGS: + value = values[flag[2:].replace("-", "_")] + if value is None: + continue + if flag in BUILDER_PATH_FLAGS: + value = str(Path(value).resolve()) + canonical += [flag, value] + canonical += [ + flag for flag in BUILDER_SWITCHES if values[flag[2:].replace("-", "_")] + ] + computed = values["computed_csv"] or ( + str(Path(values["out_dir"]) / "computed.csv") if values["out_dir"] else None + ) + return canonical, Path(computed).resolve() if computed else None + + +BASE_ENGINE = "policyengine-us 2.38.6" +BUILD_FILES = ( + "computed.csv", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", +) +NOTE = ( + "When PolicyBench began sweeping the references for the 2026-10-12 move, " + "which policyengine-us release was the newest then and at publication, and " + "the publication check. Times are UTC. The PyPI upload times were read from " + "the PyPI JSON API; the install and first-output times are filesystem birth " + "times (st_birthtime) of the engine's dist-info directory and of the first " + "file the sweep wrote." +) + + +class Refusal(SystemExit): + pass + + +def version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def birth_time(path: Path) -> float: + """The file's birth time (st_birthtime). The record dates the install and + the outputs by birth times, which macOS keeps and Linux's stat does not; + there this refuses rather than date them by a modification time.""" + try: + return path.stat().st_birthtime + except AttributeError: + raise Refusal( + f"{path}: this platform's stat keeps no birth time (st_birthtime); " + "run sweep_timing.py on macOS" + ) from None + + +def utc(timestamp: float) -> str: + return ( + datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + +def now_utc() -> str: + return utc(datetime.datetime.now(datetime.timezone.utc).timestamp()) + + +def wheel_uploads(pypi: dict) -> dict[str, str]: + """Each plain X.Y.Z release's wheel upload time (ISO 8601, UTC).""" + uploads = {} + for version, files in pypi.get("releases", {}).items(): + try: + version_key(version) + except ValueError: + continue # pre-releases and other non-numeric tags + wheels = [f for f in files if f.get("packagetype") == "bdist_wheel"] + if wheels and not all(f.get("yanked") for f in wheels): + uploads[version] = min(f["upload_time_iso_8601"] for f in wheels) + return uploads + + +def yanked_newer(pypi: dict, engine: str) -> dict[str, str]: + """Releases newer than the engine whose wheels are all yanked, with the + reason PyPI gives (wheel_uploads leaves them out).""" + yanked = {} + for version, files in pypi.get("releases", {}).items(): + try: + key = version_key(version) + except ValueError: + continue + wheels = [f for f in files if f.get("packagetype") == "bdist_wheel"] + if ( + key > version_key(engine) + and wheels + and all(f.get("yanked") for f in wheels) + ): + yanked[version] = wheels[0].get("yanked_reason") or "" + return dict(sorted(yanked.items(), key=lambda kv: version_key(kv[0]))) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def newest(uploads: dict[str, str]) -> str: + return max(uploads, key=version_key) + + +def from_engine(uploads: dict[str, str], engine: str) -> dict[str, str]: + """The upload times of the engine and every release after it.""" + return { + v: uploads[v] + for v in sorted(uploads, key=version_key) + if version_key(v) >= version_key(engine) + } + + +def newer_before(uploads: dict[str, str], engine: str, moment: str) -> list[str]: + """Releases newer than the engine uploaded before a moment.""" + return sorted( + ( + v + for v, at in uploads.items() + if version_key(v) > version_key(engine) and at[:19] < moment[:19] + ), + key=version_key, + ) + + +def read_pypi(url: str = PYPI) -> dict: + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response) + + +def dist_info(venv: Path, engine: str) -> Path: + found = sorted(venv.glob(f"lib/python*/site-packages/{PACKAGE}-{engine}.dist-info")) + if len(found) != 1: + raise Refusal(f"{venv} has no single {PACKAGE}-{engine}.dist-info") + return found[0] + + +def relative(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT)) + except ValueError: + return str(path) + + +def read_computed(path: Path) -> dict[tuple[str, str], tuple[str, float]]: + with path.open(newline="") as source: + return { + (row["scenario_id"], row["variable"]): ( + row["state"], + float(row["recomputed"]), + ) + for row in csv.DictReader(source) + } + + +def compare( + reference: dict[tuple[str, str], tuple[str, float]], + check: dict[tuple[str, str], tuple[str, float]], + reference_engine: str, + check_engine: str, + excluded: frozenset[tuple[str, str]] = frozenset(), +) -> tuple[list[dict], dict]: + """Every output on both engines, and whether the values are the same. + ``excluded`` names the outputs the release does not score: a newer release + that fixes a defect behind one moves its value without touching a scored + reference, so the summary counts scored and excluded outputs apart.""" + if set(reference) != set(check): + raise Refusal( + "the two sweeps cover different outputs: " + f"{sorted(set(reference) ^ set(check))[:5]}" + ) + rows = [] + for key in sorted(reference): + state, before = reference[key] + after = check[key][1] + if not (math.isfinite(before) and math.isfinite(after)): + raise Refusal(f"{key}: a non-finite value ({before!r}, {after!r})") + delta = after - before + rows.append( + { + "scenario_id": key[0], + "state": state, + "variable": key[1], + "reference_engine": reference_engine, + "engine": check_engine, + "reference_value": repr(before), + "value": repr(after), + "delta": repr(delta), + "same": abs(delta) <= SAME, + "scored": key not in excluded, + } + ) + unknown = sorted(excluded - set(reference)) + if unknown: + raise Refusal(f"excluded outputs the sweeps do not cover: {unknown[:5]}") + differ = [r for r in rows if not r["same"]] + scored = [r for r in rows if r["scored"]] + + def names(found: list[dict]) -> list[str]: + return [f"{r['scenario_id']}|{r['variable']}" for r in found] + + return rows, { + "outputs": len(rows), + "same": len(rows) - len(differ), + "differ": names(differ), + "scored_outputs": len(scored), + "scored_same": sum(r["same"] for r in scored), + "scored_differ": names([r for r in differ if r["scored"]]), + "excluded_differ": names([r for r in differ if not r["scored"]]), + "max_abs_delta": max((abs(float(r["delta"])) for r in rows), default=0.0), + } + + +def write_rows(path: Path, rows: list[dict]) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as out: + writer = csv.DictWriter(out, fieldnames=list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def sweep(args) -> dict: + engine = args.engine + first = Path(args.first_output).resolve() + venv = Path(args.venv).resolve() + installed = dist_info(venv, engine) + install = engine_install(venv, engine) + first_at = utc(birth_time(first)) + pypi = read_pypi() + read_at = now_utc() + uploads = wheel_uploads(pypi) + if engine not in uploads: + raise Refusal(f"PyPI lists no wheel for policyengine-us {engine}") + installed_at = utc(birth_time(installed)) + sweep_order_problems(uploads[engine], installed_at, first_at, engine) + earlier = newer_before(uploads, engine, first_at) + if earlier: + raise Refusal( + f"policyengine-us {earlier} was uploaded before the sweep's first " + f"output ({first_at}), so {engine} was not the newest release when " + "the sweep began" + ) + return { + "note": NOTE, + "pypi": { + "source": PYPI, + "read_at_utc": read_at, + "newest_at_read": newest(uploads), + "wheel_uploaded_at_utc": from_engine(uploads, engine), + "yanked_newer": yanked_newer(pypi, engine), + }, + "reference_sweep": { + "engine": engine, + "engine_installed_at_utc": installed_at, + "engine_installed_evidence": relative(installed), + "engine_install": install, + "script": ( + "reference_audit/2026-10-12-engine-upgrade/scripts/" + "build_references_upgrade.py (first pass, empty actions)" + ), + "first_output_at_utc": first_at, + "first_output": relative(first), + "first_output_sha256": sha256(first), + }, + } + + +def sweep_order_problems( + uploaded_at: str, installed_at: str, output_at: str, engine: str +) -> None: + """A sweep's output follows its engine: the wheel was on PyPI before it was + installed, and installed before the sweep wrote its output. An older file + (an earlier rehearsal's output, say) would otherwise backdate the sweep.""" + if not uploaded_at[:19] <= installed_at[:19] <= output_at[:19]: + raise Refusal( + f"policyengine-us {engine}: wheel uploaded {uploaded_at}, installed " + f"{installed_at}, output written {output_at}; an output must follow " + "its engine's install, and the install its upload" + ) + + +def build_problems( + build: Path, engine: str, snapshot: Path +) -> tuple[dict, frozenset[tuple[str, str]], dict[str, str]]: + """The reference build's computed values, its excluded outputs and the + sha256 of each build file, once the build is shown to be the release's.""" + files = {name: build / name for name in BUILD_FILES} + missing = [name for name, path in files.items() if not path.is_file()] + if missing: + raise Refusal(f"{build} lacks {missing}") + pins = {name: sha256(path) for name, path in files.items()} + meta = json.loads(files["reference_outputs.csv.meta.json"].read_text()) + revision = meta["revisions"][-1] + if ( + revision.get("kind") != "engine_upgrade" + or revision.get("engine_version") != f"policyengine-us {engine}" + ): + raise Refusal( + f"{build}'s last revision is not an engine upgrade to {engine}: " + f"{revision.get('kind')} {revision.get('engine_version')}" + ) + if meta.get("reference_csv_sha256") != pins["reference_outputs.csv"]: + raise Refusal(f"{build}'s sidecar does not pin its reference_outputs.csv") + # The check sweep runs this audit's builder, as the build did: the same + # conventions and adapter, so the sidecar must name the committed builder. + named = (revision.get("provenance") or {}).get("builder_sha256") + if named != sha256(BUILDER): + raise Refusal( + f"{build}'s sidecar names builder {named!r}, not the committed one" + ) + pins["builder"] = named + for name in ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + ): + committed = snapshot / name + if not committed.is_file() or sha256(committed) != pins[name]: + raise Refusal( + f"{build}/{name} is not the committed snapshot's ({committed}); " + "check the build the release froze" + ) + record = json.loads(files["reference_exclusions.json"].read_text())["exclusions"] + excluded = frozenset((e["scenario_id"], e["variable"]) for e in record) + with files["reference_outputs.csv"].open(newline="") as source: + references = { + (row["scenario_id"], row["variable"]): float(row["value"]) + for row in csv.DictReader(source) + } + computed = read_computed(files["computed.csv"]) + nonfinite = sorted( + key + for key in set(references) | set(computed) + if not math.isfinite(references.get(key, 0.0)) + or not math.isfinite(computed.get(key, ("", 0.0))[1]) + ) + if nonfinite: + raise Refusal(f"{build} holds non-finite values: {nonfinite[:5]}") + if set(computed) != set(references) or not excluded <= set(references): + raise Refusal( + f"{build}'s computed.csv, references and record cover different outputs" + ) + off = sorted( + key + for key, value in references.items() + if key not in excluded and abs(computed[key][1] - value) > REFERENCE_EPS + ) + if off: + raise Refusal(f"{build}'s computed.csv is not its scored references: {off[:5]}") + return computed, excluded, pins + + +# The engine check and, when a builder is named, the builder itself, in ONE +# process under the venv's interpreter, so what is checked is what computes. +# Launched with -P (no script or working directory on sys.path) and a fresh, +# empty bytecode-cache prefix, so no __pycache__ beside the sources is read: +# every module compiles from the verified source. Before policyengine_us is +# imported, it requires the expected version, an import origin inside the +# installed package, no editable install, every package file at the hash the +# wheel's RECORD gives (sha256, sha384 or sha512), and no importable file in the +# package (a module, a sourceless .pyc, a native extension) that RECORD does not +# list. It writes what it found to PB_ENGINE_RECEIPT, then runs the builder with +# the remaining arguments, or stops (exit 3) on any problem. +ENGINE_RUNNER = r""" +import base64, hashlib, importlib.machinery as mach, importlib.metadata as md +import importlib.util, json, os, pathlib, runpy, sys +expected = os.environ["PB_EXPECTED_ENGINE"] +dist = md.distribution("policyengine-us") +spec = importlib.util.find_spec("policyengine_us") +origin = pathlib.Path(spec.origin).resolve() +site = pathlib.Path(dist.locate_file("")).resolve() +package = (site / "policyengine_us").resolve() +direct = dist.read_text("direct_url.json") +editable = bool(direct and json.loads(direct).get("dir_info", {}).get("editable")) +recorded, mismatched = set(), [] +for entry in dist.files or []: + name = str(entry) + if not name.startswith("policyengine_us/") or "__pycache__" in name: + continue + recorded.add(name) + if not entry.hash or entry.hash.mode not in ("sha256", "sha384", "sha512"): + mismatched.append(name) + continue + path = pathlib.Path(dist.locate_file(entry)) + data = path.read_bytes() if path.is_file() else None + digest = hashlib.new(entry.hash.mode, data).digest() if data is not None else b"" + if base64.urlsafe_b64encode(digest).rstrip(b"=").decode() != entry.hash.value: + mismatched.append(name) +# No symlink anywhere in the package (a linked directory hides its files from +# the scan, and a linked file leaves the package): installs must be copies. +links = sorted( + str(pathlib.Path(root, name).relative_to(site)) + for root, dirs, files in os.walk(package) + for name in dirs + files + if os.path.islink(os.path.join(root, name)) +) +if os.path.islink(site / "policyengine_us"): + links.insert(0, "policyengine_us") +suffixes = tuple(mach.all_suffixes()) +unrecorded = sorted( + str(p.relative_to(site)) + for p in package.rglob("*") + if p.is_file() and "__pycache__" not in p.parts and p.name.endswith(suffixes) + and str(p.relative_to(site)) not in recorded +) +found = { + "version": dist.version, + "origin": str(origin), + "imported_from_package": origin.is_relative_to(package), + "editable": editable, + "files_verified": len(recorded) - len(mismatched), + "files_mismatched": mismatched[:5], + "files_unrecorded": unrecorded[:5], + "symlinks": links[:5], + "pycache_prefix": sys.pycache_prefix, + "sys_path": sys.path, +} +problems = [] +if found["version"] != expected: + problems.append(f"version {found['version']}, not {expected}") +if not found["imported_from_package"]: + problems.append(f"imports policyengine_us from {found['origin']}") +if editable: + problems.append("an editable install") +if mismatched or not found["files_verified"]: + problems.append(f"files unlike the wheel's RECORD {mismatched[:5]}") +if unrecorded: + problems.append(f"importable files the wheel lacks {unrecorded[:5]}") +if links: + problems.append(f"symlinks in the package (installs must be copies) {links[:5]}") +if not sys.pycache_prefix or any(pathlib.Path(sys.pycache_prefix).iterdir()): + problems.append("no fresh bytecode-cache prefix") +found["problems"] = problems +pathlib.Path(os.environ["PB_ENGINE_RECEIPT"]).write_text(json.dumps(found)) +if problems: + sys.exit(3) +if len(sys.argv) > 1: + sys.argv = sys.argv[1:] + runpy.run_path(sys.argv[0], run_name="__main__") +""" + + +def sweep_env() -> dict[str, str]: + """The environment a sweep runs in: no inherited Python overrides.""" + return { + **{k: v for k, v in os.environ.items() if k in ("PATH", "HOME", "LANG")}, + "PYTHONPATH": str(ROOT), + "PYTHONDONTWRITEBYTECODE": "1", + "OPENBLAS_NUM_THREADS": "1", + } + + +def run_on_engine( + venv: Path, engine: str, receipt: Path, builder_args: list[str] | None = None +): + """ENGINE_RUNNER under the venv's interpreter: the engine check, then (with + ``builder_args``) the builder in the same process. Returns the completed + process and the check's record; refuses when the check found a problem.""" + receipt = receipt.resolve() + if receipt.exists(): + raise Refusal(f"{receipt} exists: each run writes a fresh receipt") + cache = Path(tempfile.mkdtemp(prefix="pb-pycache-")) + command = [str(venv / "bin" / "python"), "-P", "-c", ENGINE_RUNNER] + if builder_args is not None: + command += [str(BUILDER), *builder_args] + env = { + **sweep_env(), + "PYTHONPYCACHEPREFIX": str(cache), + "PB_EXPECTED_ENGINE": engine, + "PB_ENGINE_RECEIPT": str(receipt), + } + try: + ran = subprocess.run(command, env=env, cwd=ROOT, capture_output=True, text=True) + finally: + shutil.rmtree(cache, ignore_errors=True) + if not receipt.is_file(): + raise Refusal( + f"{venv}: the engine check did not run ({ran.stderr.strip()[-300:]})" + ) + found = json.loads(receipt.read_text()) + if found["problems"]: + raise Refusal( + f"{venv} is not policyengine-us {engine} as released: {found['problems']}" + ) + return ran, { + "version": found["version"], + "files_verified": found["files_verified"], + "editable": False, + "imported_from_package": True, + "fresh_bytecode_cache": True, + } + + +def engine_install(venv: Path, engine: str) -> dict: + """The engine check alone (ENGINE_RUNNER without a builder).""" + with tempfile.TemporaryDirectory() as scratch: + _, install = run_on_engine(venv, engine, Path(scratch) / "receipt.json") + return install + + +def run_check_sweep(venv: Path, engine: str, out_dir: Path) -> tuple[Path, dict]: + """The check sweep, run here: this audit's builder, a first pass with + empty actions naming ``engine``, in the same process as the engine check + (run_on_engine), under the venv's interpreter. The builder also refuses + before computing unless the venv holds that release, and it pins the + conventions and harness it applies, so the computed.csv it writes is that + engine's under the release's conventions. Returns the file and what ran.""" + out_dir.mkdir(parents=True, exist_ok=False) + actions = out_dir / "actions.empty.json" + actions.write_text( + json.dumps( + { + "engine": f"policyengine-us {engine}", + "previous_engine": BASE_ENGINE, + "date": now_utc()[:10], + "release_spec_sha256": sha256(SPEC), + "draft": True, + } + ) + ) + args = ["--actions", str(actions), "--out-dir", str(out_dir), "--allow-draft"] + ran, install = run_on_engine(venv, engine, out_dir / "engine_install.json", args) + (out_dir / "sweep.log").write_text(ran.stdout + ran.stderr) + computed = out_dir / "computed.csv" + if not computed.is_file(): + raise Refusal( + f"the check sweep on {engine} wrote no computed.csv " + f"(exit {ran.returncode}; see {out_dir / 'sweep.log'})" + ) + return computed, { + "builder_sha256": sha256(BUILDER), + "venv": relative(venv), + "command": "build_references_upgrade.py --actions --allow-draft", + "exit_code": ran.returncode, + "engine_install": install, + } + + +def check(args) -> dict: + timing = json.loads(TIMING.read_text()) + engine = timing["reference_sweep"]["engine"] + build = Path(args.build) + computed, excluded, pins = build_problems( + build, engine, Path(getattr(args, "snapshot", None) or SNAPSHOT) + ) + scored = len(computed) - len(excluded) + pypi = read_pypi() + read_at = now_utc() + uploads = wheel_uploads(pypi) + latest = newest(uploads) + timing["pypi"] = { + "source": PYPI, + "read_at_utc": read_at, + "newest_at_read": latest, + "wheel_uploaded_at_utc": from_engine(uploads, engine), + "yanked_newer": yanked_newer(pypi, engine), + } + inputs = { + "build": relative(build), + "build_sha256": pins, + "scored_outputs": scored, + } + if latest == engine: + timing["publication_check"] = { + "engine": engine, + "result": ( + f"policyengine-us {engine}, the reference engine, was still the " + "newest release when PolicyBench read PyPI" + ), + **inputs, + } + return timing + if not args.check_venv: + raise Refusal( + f"policyengine-us {latest} is newer than the reference engine {engine}: " + "pass --check-venv, a venv holding it, and check runs the sweep" + ) + venv = Path(args.check_venv).resolve() + installed = dist_info(venv, latest) + engine_install(venv, latest) # fail fast; the sweep's own process checks again + installed_at = utc(birth_time(installed)) + scratch = ROOT / "results/local" + scratch.mkdir(parents=True, exist_ok=True) + work = ( + Path( + getattr(args, "work_dir", None) + or tempfile.mkdtemp(prefix=f"check-sweep-{latest}-", dir=scratch) + ).resolve() + / "sweep" + ) + check_path, receipt = run_check_sweep(venv, latest, work) + output_at = utc(birth_time(check_path)) + sweep_order_problems(uploads[latest], installed_at, output_at, latest) + rows, summary = compare( + computed, read_computed(check_path), engine, latest, excluded + ) + if summary["scored_outputs"] != scored: + raise Refusal( + f"the check scored {summary['scored_outputs']} outputs, not {scored}" + ) + out = VERIFICATION / f"publication_check_{latest.replace('.', '_')}.csv" + digest = write_rows(out, rows) + timing["publication_check"] = { + "engine": latest, + "engine_installed_at_utc": installed_at, + "engine_installed_evidence": relative(installed), + "check_computed": relative(check_path), + "check_computed_sha256": sha256(check_path), + "check_sweep": receipt, + "output_at_utc": output_at, + "output": relative(out), + "output_sha256": digest, + **inputs, + **summary, + } + return timing + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="step", required=True) + s = sub.add_parser("sweep") + s.add_argument("--engine", required=True) + s.add_argument("--venv", required=True) + s.add_argument("--first-output", required=True) + c = sub.add_parser("check") + c.add_argument("--build", required=True) + c.add_argument("--check-venv") + r = sub.add_parser("run") + r.add_argument("--venv", required=True) + r.add_argument("--engine", required=True) + r.add_argument("--receipt", required=True) + r.add_argument("builder_args", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + if args.step == "run": + # The reference build itself, through the same engine check and the + # same single process as the check sweep (run_on_engine). + raw = list(args.builder_args) + if raw[:1] == ["--"]: + raw = raw[1:] + builder_args, computed = builder_arguments(raw) + if computed is not None and computed.exists(): + raise Refusal(f"{computed} exists: each run writes into a fresh out-dir") + started = time.time() + ran, install = run_on_engine( + Path(args.venv).resolve(), args.engine, Path(args.receipt), builder_args + ) + if computed is not None and ( + not computed.is_file() or birth_time(computed) < started - 1 + ): + raise Refusal( + f"the build on {args.engine} wrote no new {computed} " + f"(exit {ran.returncode})" + ) + sys.stdout.write(ran.stdout) + sys.stderr.write(ran.stderr) + print(json.dumps({"engine_install": install}), file=sys.stderr) + raise SystemExit(ran.returncode) + timing = sweep(args) if args.step == "sweep" else check(args) + VERIFICATION.mkdir(parents=True, exist_ok=True) + TIMING.write_text(json.dumps(timing, indent=2) + "\n") + print( + json.dumps(timing.get("publication_check", timing["reference_sweep"]), indent=1) + ) + moved = timing.get("publication_check", {}).get("scored_differ") + if moved: + raise Refusal( + f"policyengine-us {timing['publication_check']['engine']} moves scored " + f"outputs {moved[:5]}: do not publish (the record is in " + f"{relative(TIMING)})" + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/date_musewonder_judge_verdicts.py b/scripts/date_musewonder_judge_verdicts.py index 64a6d916..d0e77324 100644 --- a/scripts/date_musewonder_judge_verdicts.py +++ b/scripts/date_musewonder_judge_verdicts.py @@ -1,29 +1,27 @@ -"""Record the verdicts release dashboard-data-20261010's adjudications name. +"""Record the verdicts release dashboard-data-20261013's adjudications name. -The Claude Haiku 5.5 release restated the judge fields of every decision on a -case Claude Haiku 5.5 re-opened (scripts/restate_gpt61sol_adjudications.py), -dropped the decisions on records its engine upgrade regenerated, and added -two waves of decisions: the 2026-10-06 rulings' (d1022, d994) and the engine -upgrade's, on the outputs it newly excludes. Run on the exported stage after -the freeze, this script +Adapted from scripts/date_haiku55_judge_verdicts.py. The Muse Wonder release +restated the judge fields of every decision on a case Muse Wonder or its +engine upgrade re-opened (scripts/restate_gpt61sol_adjudications.py), dropped +the decisions on records its engine upgrade regenerated, and adds a wave of +decisions only for outputs the upgrade newly excludes. Run on the exported +stage after the freeze, this script -- writes docs/haiku55/judge_verdicts_20261010.json: for each committed +- writes docs/musewonder/judge_verdicts_.json: for each committed decision on a re-opened case, the case's verdict in the stage, read from - verdict.json and its sha256-bound verdict.meta.json, as - scripts/date_gpt61sol_judge_verdicts.py did for release 20260930; and + verdict.json and its sha256-bound verdict.meta.json, as release 20261010's + did; and - brings reference_audit/2026-09-28/verification/judge_verdicts.json up to - this release, as release 20261006's driver did - (scripts/release_20261006.py build_evidence): the 2026-10-05 wave's release - is now committed (the driver's BASE_COMMIT), so its decisions gain the - verdicts that release published; this release's waves join, uncommitted, - with the UTC day their decisions were written; and each decision of theirs, - new or restated in place from an earlier wave, gains the stage verdict it - reviewed in place of the earlier evidence. + this release: the 2026-10-06 and 2026-10-10 waves' release (20261010) is now + committed (the driver's BASE_COMMIT), so their decisions gain the verdicts + that release published; any wave of this release's joins, uncommitted, with + the UTC day its decisions were written; and each decision of its gains the + stage verdict it reviewed in place of the earlier evidence. tests/test_adjudications.py reads both files on machines without the stage. A second run on the same stage writes the same bytes. - python scripts/date_haiku55_judge_verdicts.py --stage-dir STAGE + python scripts/date_musewonder_judge_verdicts.py --stage-dir STAGE """ from __future__ import annotations @@ -41,16 +39,17 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) -import finish_haiku55 as driver # noqa: E402 +import finish_musewonder as driver # noqa: E402 from date_adds0928_judge_verdicts import _evidence # noqa: E402 # Named for the release tag's date (dashboard-data-YYYYMMDD). -OUT = ROOT / f"docs/haiku55/judge_verdicts_{driver.RELEASE_TAG[-8:]}.json" +OUT = ROOT / f"docs/musewonder/judge_verdicts_{driver.RELEASE_TAG[-8:]}.json" EVIDENCE = ROOT / "reference_audit/2026-09-28/verification/judge_verdicts.json" -# The wave release 20261006 decided, which this release's base commit froze. -FILL_WAVE = "2026-10-05" -# Release 20261006's waves, before this release's. -BASE_WAVES = ["2026-09-05", "2026-09-22", "2026-09-29", FILL_WAVE] +# The waves release 20261010 decided, which this release's base commit froze. +FILL_WAVES = ["2026-10-06", "2026-10-10"] +FILL_RELEASE = "dashboard-data-20261010" +# Release 20261010's waves, before this release's. +BASE_WAVES = ["2026-09-05", "2026-09-22", "2026-09-29", "2026-10-05", *FILL_WAVES] def case_id(entry: dict) -> str: @@ -133,12 +132,12 @@ def restatements(stage: Path, record: dict) -> dict: } return { "note": ( - "The verdicts release dashboard-data-20261010's adjudication entries " - "name on the cases Claude Haiku 5.5 re-opened: for each committed " - "decision on such a case, the case's verdict in the exported stage, " - "read by scripts/date_haiku55_judge_verdicts.py from verdict.json and " - "its sha256-bound verdict.meta.json. The verdict each restatement " - "replaced is release 20261006's entry (git show " + f"The verdicts release {driver.RELEASE_TAG}'s adjudication entries " + "name on the cases Muse Wonder or the engine upgrade re-opened: for " + "each committed decision on such a case, the case's verdict in the " + "exported stage, read by scripts/date_musewonder_judge_verdicts.py " + "from verdict.json and its sha256-bound verdict.meta.json. The " + "verdict each restatement replaced is release 20261010's entry (git show " f"{driver.BASE_COMMIT[:12]}:annotations/{driver.RUN_NAME}/" f"{driver.ADJUDICATIONS})." ), @@ -149,8 +148,9 @@ def restatements(stage: Path, record: dict) -> dict: def fill_release() -> dict: - """The 2026-10-05 wave's release as committed: the base commit, its pull - request (the squash subject's "(#N)") and its UTC commit day.""" + """The 2026-10-06 and 2026-10-10 waves' release as committed: the base + commit, its pull request (the squash subject's "(#N)") and its UTC commit + day.""" subject = git("show", "-s", "--format=%s", driver.BASE_COMMIT).strip() number = re.search(r"\(#(\d+)\)$", subject) driver.require(number is not None, f"{subject!r} names no pull request") @@ -159,7 +159,7 @@ def fill_release() -> dict: ) return { "commit": driver.BASE_COMMIT, - "release": "dashboard-data-20261006", + "release": FILL_RELEASE, "pull_request": int(number.group(1)), "committed_on": when.astimezone(datetime.timezone.utc).date().isoformat(), } @@ -167,7 +167,7 @@ def fill_release() -> dict: def upgrade_day() -> str | None: """The day of the committed sidecar's engine upgrade, if this release has one - (the last revision, after release 20261006's 2026-09-29 move).""" + (the last revision, after release 20261010's 2026-10-10 move).""" meta = json.loads((driver.SNAPSHOT / "reference_outputs.csv.meta.json").read_text()) last = meta["revisions"][-1] if last.get("kind") == "engine_upgrade" and last["date"] > BASE_WAVES[-1]: @@ -177,36 +177,33 @@ def upgrade_day() -> str | None: def new_waves(record: dict, upgrade: str | None = None) -> dict[str, str]: """This release's waves and the UTC day each one's decisions were written - (the spec's adjudications_written_on for the 2026-10-06 rulings, its - upgrade_adjudications_written_on for the engine upgrade's wave). The waves - must be the 2026-10-06 rulings' and, with an engine upgrade, its day.""" + (the spec's upgrade_adjudications_written_on for the engine upgrade's + wave). The release rules nothing, so its only possible wave is the engine + upgrade's, on the outputs it newly excludes; there may be none.""" spec = driver.load_spec() waves = sorted( {e["adjudicated_on"] for e in record["adjudications"]} - set(BASE_WAVES) ) - allowed = {"2026-10-06"} | ({upgrade} if upgrade else set()) + allowed = {upgrade} if upgrade else set() driver.require( - "2026-10-06" in waves and set(waves) <= allowed, - f"this release's waves are {waves}, not the 2026-10-06 rulings' and the " - f"engine upgrade's ({sorted(allowed)})", + set(waves) <= allowed, + f"this release's waves are {waves}, not the engine upgrade's " + f"({sorted(allowed)})", ) written = {} for wave in waves: - if wave == "2026-10-06": - written[wave] = spec["adjudications_written_on"] - else: - written[wave] = spec["upgrade_adjudications_written_on"] + written[wave] = spec["upgrade_adjudications_written_on"] driver.require(wave <= written[wave], f"the {wave} wave was written before it") return written def base_wave_releases(waves: dict) -> dict: - """Release 20261006's waves alone. The waves after them are this release's, + """Release 20261010's waves alone. The waves after them are this release's, which every pass writes afresh from the record, so an earlier pass's (a rehearsal's, or one whose record has since changed) cannot outlive it.""" driver.require( list(waves)[: len(BASE_WAVES)] == BASE_WAVES, - "the evidence's waves are not release 20261006's", + "the evidence's waves are not release 20261010's", ) return {wave: waves[wave] for wave in BASE_WAVES} @@ -215,15 +212,14 @@ def updated_evidence(evidence: dict, stage: Path, record: dict) -> dict: evidence = copy.deepcopy(evidence) waves = base_wave_releases(evidence["wave_releases"]) fill = fill_release() - if waves[FILL_WAVE].get("commit") is None: - driver.require( - waves[FILL_WAVE]["release"] == fill["release"], - f"the {FILL_WAVE} wave is not release 20261006's", - ) - waves[FILL_WAVE] = fill - driver.require( - waves[FILL_WAVE] == fill, f"the {FILL_WAVE} wave was filled otherwise" - ) + for wave in FILL_WAVES: + if waves[wave].get("commit") is None: + driver.require( + waves[wave]["release"] == fill["release"], + f"the {wave} wave is not release 20261010's", + ) + waves[wave] = fill + driver.require(waves[wave] == fill, f"the {wave} wave was filled otherwise") published = json.loads( git( "show", @@ -232,7 +228,7 @@ def updated_evidence(evidence: dict, stage: Path, record: dict) -> dict: )["adjudications"] filled = 0 for entry in published: - if entry["adjudicated_on"] != FILL_WAVE: + if entry["adjudicated_on"] not in FILL_WAVES: continue evidence["cases"][case_id(entry)]["published"] = { "release": fill["release"], @@ -243,7 +239,7 @@ def updated_evidence(evidence: dict, stage: Path, record: dict) -> dict: "judged_on_utc": entry.get("judged_on_utc"), } filled += 1 - driver.require(filled > 0, f"release 20261006 published no {FILL_WAVE} decisions") + driver.require(filled > 0, f"release 20261010 published no {FILL_WAVES} decisions") for wave, written in new_waves(record, upgrade_day()).items(): waves[wave] = { "commit": None, diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py index e6fa148a..0618d1e5 100644 --- a/scripts/finish_musewonder.py +++ b/scripts/finish_musewonder.py @@ -101,6 +101,7 @@ "judge_effort", "judge_model_reported", "judged_at_utc", + "judge_template_version", "prompt_sha256", "verdict_sha256", } @@ -112,6 +113,7 @@ "judge_model_reported", "judged_at_utc", "judge_account_declared", + "judge_template_version", ) WITHHELD_ADDRESS = "" # The context attachments an isolated judge's transcript may carry: ATTACHMENTS @@ -2420,7 +2422,8 @@ def validate_verdicts( whose prompt is the seed's (``seed``, as stage.json binds it) carries its seed verdict over: the verdict must be the seed's, byte for byte, and any prompt_sha256 its sidecar records must match. Every other verdict is new - and must record the sha256 of the prompt it judged. A verdict naming + and must record the sha256 of the prompt it judged and template + JUDGE_TEMPLATE_VERSION. A verdict naming Muse Wonder also needs bound Opus 5.5 provenance. Returns the pending cases; ``remove_invalid`` sets their verdicts aside. A carried-over verdict that fails is refused outright: a re-judge cannot restore it. @@ -2465,6 +2468,12 @@ def validate_verdicts( raise ValueError("verdict is bound to a different prompt") elif bound != digest(prompt): raise ValueError("a new verdict is not bound to this prompt") + elif meta.get("judge_template_version") != JUDGE_TEMPLATE_VERSION: + # A new or re-opened case is rendered on this release's + # template, and its judge must have read that version. + raise ValueError( + f"a new verdict is not on judge template v{JUDGE_TEMPLATE_VERSION}" + ) if set(names) & set(MODELS.values()): if ( meta.get("judge_model_requested") != JUDGE_MODEL @@ -3395,8 +3404,25 @@ def parts(engine) -> tuple[int, ...] | None: # The audited root-cause fix modules a "fix_modules" regeneration target -# names, as committed at BASE_COMMIT. +# names, as committed at BASE_COMMIT; and the upgrade audit's own modules, for +# defects found later, as committed at HEAD (the builder's fix_location). A +# target names such a module's path. AUDIT_FIXES = Path("reference_audit/2026-09-22/fixes") +LOCAL_FIXES = Path("reference_audit/2026-10-12-engine-upgrade/fixes") + + +def fix_module_blob(name: str) -> tuple[str, bytes]: + """A fix module's repository path and committed bytes: under AUDIT_FIXES + at BASE_COMMIT, or, for one the upgrade audit adds (absent there), under + LOCAL_FIXES at HEAD.""" + path = AUDIT_FIXES / name + result = subprocess.run( + ["git", "-C", str(ROOT), "cat-file", "-e", f"{BASE_COMMIT}:{path.as_posix()}"], + capture_output=True, + ) + if result.returncode == 0: + return path.as_posix(), base_commit_blob(path) + return (LOCAL_FIXES / name).as_posix(), head_blob((LOCAL_FIXES / name).as_posix()) TARGET_KEYS = { "record": {"kind", "value", "engine"}, "fix_modules": { @@ -3414,8 +3440,8 @@ def parts(engine) -> tuple[int, ...] | None: def committed_dependencies(names: list[str]) -> list[str]: - """The siblings the named fix modules load, transitively, as committed at - BASE_COMMIT, in the builder's order: the builder's module_dependencies + """The siblings the named fix modules load, transitively, as committed + (fix_module_blob), in the builder's order: the builder's module_dependencies (policybench.fix_module_closure), re-derived here from the committed bytes. Raises ClosureError for a module that loads local files any way but Path(__file__).with_name("").""" @@ -3423,7 +3449,7 @@ def committed_dependencies(names: list[str]) -> list[str]: found: list[str] = [] for name in names: - read = lambda file: base_commit_blob(AUDIT_FIXES / file) # noqa: E731 + read = lambda file: fix_module_blob(file)[1] # noqa: E731 for dep in dependency_closure(name, read): if dep not in names and dep not in found: found.append(dep) @@ -3502,7 +3528,7 @@ def regeneration_target_problems( A "record" target is the record's alternative_value on the record's engine. A "fix_modules" target is the corrected value in a committed evidence file, at its sha256, computed on an older engine than the build's - with fix modules committed at BASE_COMMIT; there the modules moved the + with committed fix modules (fix_module_blob); there the modules moved the output beyond the exact-match tolerance (the defect was present), and on the build's engine they moved it by no more than REGENERATION_TOLERANCE (nothing is left to fix). Either way the build's value lands on it. @@ -3572,11 +3598,18 @@ def _fix_modules_target_problems( f"or is not older than the build's {engine}" ) modules = target["modules"] - if not ( - isinstance(modules, list) - and modules - and all(isinstance(m, dict) and set(m) == {"module", "sha256"} for m in modules) - ): + + def pinned(item) -> bool: + # A module the upgrade audit adds names its path; no other does. + if not isinstance(item, dict) or not isinstance(item.get("module"), str): + return False + local = fix_module_blob(item["module"])[0].startswith(LOCAL_FIXES.as_posix()) + keys = {"module", "sha256", "path"} if local else {"module", "sha256"} + return set(item) == keys and ( + not local or item["path"] == (LOCAL_FIXES / item["module"]).as_posix() + ) + + if not (isinstance(modules, list) and modules and all(map(pinned, modules))): return [*problems, f"{key}: the target lists no fix modules"] names = [m["module"] for m in modules] from policybench.fix_module_closure import ClosureError @@ -3587,29 +3620,24 @@ def _fix_modules_target_problems( return [*problems, f"{key}: {error}"] expected = [] for dep in closure: - expected.append( - { - "module": dep, - "sha256": hashlib.sha256( - base_commit_blob(AUDIT_FIXES / dep) - ).hexdigest(), - } - ) + where, blob = fix_module_blob(dep) + item = {"module": dep, "sha256": hashlib.sha256(blob).hexdigest()} + if where.startswith(LOCAL_FIXES.as_posix()): + item["path"] = where + expected.append(item) if target["dependencies"] != expected or any( doc["modules"].get(item["module"]) != item["sha256"] for item in expected ): problems.append( f"{key}: the target and evidence do not pin the siblings its modules load " - f"as committed at {BASE_COMMIT[:12]}: {[i['module'] for i in expected]}" + f"as committed: {[i['module'] for i in expected]}" ) for entry in modules: name = entry["module"] - committed = hashlib.sha256(base_commit_blob(AUDIT_FIXES / name)).hexdigest() + where, blob = fix_module_blob(name) + committed = hashlib.sha256(blob).hexdigest() if entry["sha256"] != committed or doc["modules"].get(name) != committed: - problems.append( - f"{key}: {name} is not {AUDIT_FIXES / name} as committed at " - f"{BASE_COMMIT[:12]}" - ) + problems.append(f"{key}: {name} is not {where} as committed") items = [ item for item in doc.get("items", []) @@ -4351,21 +4379,27 @@ def object_strings(): def render_audit_with( - bundle: Path, sources: dict[str, bytes], lookup: dict -) -> list[tuple[str, str, dict]]: - """Each audit case (id, prompt, manifest row) the bundle gives once the - build's references and explanations are in it, rendered in memory. + bundle: Path, + sources: dict[str, bytes], + lookup: dict, + versions: dict[str, int] | None = None, +) -> list[tuple[str, dict[int, str], dict]]: + """Each audit case (id, prompts by template version, manifest row) the + bundle gives once the build's references and explanations are in it, + rendered in memory. A scratch run directory links every other bundle file and holds the build's, so nothing in the stage changes. The order is prepare_audit's. - Every prompt is on JUDGE_TEMPLATE_VERSION, the version of every verdict - this release's stage holds; install-references refuses a stage whose - prepare_audit rendering differs. + Every case is rendered on JUDGE_TEMPLATE_VERSION, the version new and + re-opened cases take, and a judged case also on the version its sidecar + records (``versions``), as prepare_audit compares it: the stage's audit + mixes the seed's v1 verdicts with this release's v2 ones. """ import tempfile from policybench.audit import build_audit_cases, render_case_prompt + versions = versions or {} with tempfile.TemporaryDirectory(prefix="install-references-") as scratch: run = Path(scratch) / bundle.name for part in ("us", "annotations"): @@ -4386,13 +4420,40 @@ def render_audit_with( return [ ( case.case_id, - render_case_prompt(case, template_version=JUDGE_TEMPLATE_VERSION), + { + version: render_case_prompt(case, template_version=version) + for version in { + JUDGE_TEMPLATE_VERSION, + versions.get(case.case_id, JUDGE_TEMPLATE_VERSION), + } + }, case.to_manifest_row(), ) for case in cases ] +def recorded_versions(audit: Path) -> dict[str, int]: + """The template version each judged case's sidecar records (absent: v1), + for the cases that hold a verdict; a sidecar that records an unknown + version is left out, so its case renders on JUDGE_TEMPLATE_VERSION.""" + from policybench.judge_template import recorded_template_version + + found = {} + for case in sorted((audit / "cases").glob("*")): + meta_path = case / "verdict.meta.json" + if not (case / "verdict.json").is_file() or not meta_path.is_file(): + continue + try: + meta = json.loads(meta_path.read_text()) + except ValueError: + continue + version = recorded_template_version(meta if isinstance(meta, dict) else None) + if version is not None: + found[case.name] = version + return found + + def verify_install_target(stage: Path, receipt: dict) -> None: """The stage's references are release 20261010's (its exclusion record perhaps the release's, from install-exclusions) or an earlier install's, @@ -4517,22 +4578,11 @@ def install_references(args) -> None: bundle = stage / "publish" / RUN_NAME audit = stage / "audit" lookup = grounding_lookup(args.grounding) - rendered = render_audit_with(bundle, sources, lookup) - texts = { - case: text for case, text, row in rendered if not row["parse_failure_only"] + versions = recorded_versions(audit) + rendered = render_audit_with(bundle, sources, lookup, versions) + renders = { + case: texts for case, texts, row in rendered if not row["parse_failure_only"] } - seeded = {case: item["prompt_sha256"] for case, item in seed.items()} - upgraded = upgrade.upgraded_cases - changes = classify_prompt_changes( - [row for _, _, row in rendered], - { - case: hashlib.sha256(text.encode()).hexdigest() - for case, text in texts.items() - }, - seeded, - reworded_since_seed(), - upgraded, - ) reopen, kept = {}, {} for case in sorted((audit / "cases").iterdir()): if not case.is_dir() or not ( @@ -4540,12 +4590,16 @@ def install_references(args) -> None: ): continue prompt = case / "prompt.md" - if case.name not in texts: + judged_on = versions.get(case.name, JUDGE_TEMPLATE_VERSION) + if case.name not in renders: reopen[case.name] = ( f"install-references: the case leaves the audit on " f"{upgrade.engine_version}" ) - elif not prompt.is_file() or prompt.read_text() != texts[case.name]: + elif ( + not prompt.is_file() + or prompt.read_bytes() != renders[case.name][judged_on].encode("utf-8") + ): reopen[case.name] = ( "install-references: its prompt re-renders on " f"{upgrade.engine_version}, whose reference value or explanation " @@ -4553,6 +4607,26 @@ def install_references(args) -> None: ) elif (case / "verdict.json").is_file(): kept[case.name] = digest(case / "verdict.json") + # The prompt each case will hold after prepare_audit: a kept verdict's own + # bytes, on its recorded version; every other case on JUDGE_TEMPLATE_VERSION. + texts = { + case: forms[versions.get(case, JUDGE_TEMPLATE_VERSION)] + if case in kept + else forms[JUDGE_TEMPLATE_VERSION] + for case, forms in renders.items() + } + seeded = {case: item["prompt_sha256"] for case, item in seed.items()} + upgraded = upgrade.upgraded_cases + changes = classify_prompt_changes( + [row for _, _, row in rendered], + { + case: hashlib.sha256(text.encode()).hexdigest() + for case, text in texts.items() + }, + seeded, + reworded_since_seed(), + upgraded, + ) # Install the build and bind it before the audit is touched: a stage left # partway is resumed by installing the same build again. @@ -4595,7 +4669,7 @@ def install_references(args) -> None: require( manifest == [row for _, _, row in rendered] and all( - (audit / "cases" / case / "prompt.md").read_text() == text + (audit / "cases" / case / "prompt.md").read_bytes() == text.encode("utf-8") for case, text in texts.items() ), "prepare_audit rendered the stage's audit unlike the in-memory rendering", diff --git a/scripts/freeze_musewonder.py b/scripts/freeze_musewonder.py index cb55e3ac..4d4f1162 100644 --- a/scripts/freeze_musewonder.py +++ b/scripts/freeze_musewonder.py @@ -1,21 +1,21 @@ -"""Build the Claude Haiku 5.5 release locally from a strictly gated stage. +"""Build the Muse Wonder release locally from a strictly gated stage. -Adapted from freeze_gpt61sol.py. It configures the existing freezer in process -and never calls a release upload. Run only after finish_haiku55.py --step +Adapted from freeze_haiku55.py. It configures the existing freezer in process +and never calls a release upload. Run only after finish_musewonder.py --step export has written release-ready.json. The optional --dry-run validates inputs without changing repository files. Both rebuild the payload from the bound bundle, as export builds it, and refuse a staged payload that differs. -The release folds one model onto release 20261006 and installs the ten -exclusions Max ruled on 2026-10-06 (docs/haiku55/spec.json). Without an engine -upgrade, references stay release 20261006's and the exclusion record is that -release's plus the ten ruled records. An installed upgrade must still pass -the driver's build and drop-log gates; its references, explanations and -exclusions are frozen byte for byte. Scenarios always stay 20261006's. The +The release folds one model onto release 20261010 and rules no exclusion +(docs/musewonder/spec.json). Without an engine upgrade, references stay +release 20261010's and so does the exclusion record. An installed upgrade must +pass the driver's build and drop-log gates; its references, explanations and +exclusions are frozen byte for byte. Scenarios always stay 20261010's. The adjudications may change only through a staged record that triage applied and -export bound: the ruled outputs' entries, a re-opened case's judge fields, and -date conventions that name the 2026-10-06 wave. The committed wording -amendments record stays release 20261006's, with any stage list appended. +export bound: a re-opened case's judge fields, the upgrade's dropped and added +decisions, and date conventions that name any wave the release adds. The +committed wording amendments record stays release 20261010's, with any stage +list appended. """ from __future__ import annotations @@ -38,14 +38,13 @@ sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / "scripts")) -import finish_haiku55 as driver # noqa: E402 +import finish_musewonder as driver # noqa: E402 # The treatment check is release-independent; reuse it unchanged. from freeze_adds0928 import validate_treatment # noqa: E402 -# Reuse release 20261006's manifest comparison and baseline version counts. +# Reuse release 20261010's manifest comparison and baseline version counts. from release_20261006 import leaf_paths # noqa: E402 -from release_20261006 import versions_description as base_count_exclusions # noqa: E402 RUN = driver.RUN_NAME NEW_MODELS = dict(driver.MODELS) @@ -53,7 +52,7 @@ ADJUDICATIONS = "us_adjudications.json" # The published annotations the adjudications and amendments are applied to. ANNOTATION_CSVS = ("us_audit_row_annotations.csv", "us_case_notes.csv") -# The reference narratives a judge's prompt carries: release 20261006's, +# The reference narratives a judge's prompt carries: release 20261010's, # unless an installed engine upgrade regenerates them. EXPLANATIONS = "us_case_reference_explanations.csv" # The wording amendments record, as it is committed beside the adjudications. @@ -67,10 +66,15 @@ ANNOTATIONS = Path("annotations") / RUN POINTER = Path("app/src/data.artifact.json") VERSIONS = Path("app/src/data.versions.json") -# The engines the exclusion records were decided on, which the live version's -# description counts by name. -EXCLUSION_ENGINES = frozenset({"policyengine-us 1.755.4", "policyengine-us 2.15.17"}) -# Manifest paths the freeze may change from release 20261006's; every other +# The engines release 20261010's exclusion records were decided on, in the +# order the live version's description counts them by name. +BASE_EXCLUSION_ENGINES = ( + "policyengine-us 1.755.4", + "policyengine-us 2.15.17", + "policyengine-us 2.38.6", +) +EXCLUSION_ENGINES = frozenset(BASE_EXCLUSION_ENGINES) +# Manifest paths the freeze may change from release 20261010's; every other # leaf must equal that release's manifest, read from git at BASE_COMMIT. MANIFEST_CHANGES = ( ("audit_annotation_artifacts", "files"), @@ -93,7 +97,7 @@ ("snapshot_date",), ("source_run_artifacts", RUN, "files"), ) -# Pins under those paths that must stay release 20261006's all the same. +# Pins under those paths that must stay release 20261010's all the same. MANIFEST_PINNED = frozenset( { *(("source_run_artifacts", RUN, "files", name) for name in PINNED_REFERENCES), @@ -102,7 +106,7 @@ ) # Additional leaves written from an installed build's sidecar. Each is named # explicitly; the scenario pins, household dataset and other refresh fields -# remain release 20261006's, including row_count and generated_at_utc. +# remain release 20261010's, including row_count and generated_at_utc. UPGRADE_MANIFEST_CHANGES = ( ("committed_snapshot_artifacts", "us_reference_outputs.csv"), ("reference_output_refresh", "date"), @@ -182,12 +186,13 @@ def verify_freezer_destinations(freezer) -> None: def release_spec(tag: str) -> dict: - """docs/haiku55/spec.json, with what the freeze reads from it checked. + """docs/musewonder/spec.json, with what the freeze reads from it checked. - The snapshot date and the day the 2026-10-06 wave's decisions were written - come from the spec alone, so a spec that lacks either stops the freeze - here, before any gate reads it. The spec must also name the release this - freeze builds and the base the driver pins. + The snapshot date comes from the spec alone, and so does the day a ruled + wave's decisions were written when the spec rules any, so a spec that + lacks either stops the freeze here, before any gate reads it. The spec + must also name the release this freeze builds and the base the driver + pins. """ spec = driver.load_spec() try: @@ -198,10 +203,10 @@ def release_spec(tag: str) -> dict: f"{spec.get('snapshot_date')!r}" ) from None written = spec.get("adjudications_written_on") - if not isinstance(written, str) or not written.strip(): + if driver.spec_records(spec) and (not isinstance(written, str) or not written.strip()): raise SystemExit( f"{driver.SPEC_PATH} names no adjudications_written_on, the UTC day " - "the 2026-10-06 wave's decisions were written; the record's date " + "the ruled wave's decisions were written; the record's date " "conventions state it" ) named = { @@ -256,7 +261,7 @@ def verify_upgrade(stage: Path, receipt: dict) -> driver.Upgrade | None: for name in ("scenarios.csv", "scenarios.csv.meta.json"): path = stage / "scoring" / name if not path.is_file() or digest(path) != driver.BASE_REFERENCE_SHA256[name]: - raise SystemExit(f"Staged scoring/{name} is not release 20261006's") + raise SystemExit(f"Staged scoring/{name} is not release 20261010's") superseded = installed.get("superseded") if ( not isinstance(superseded, list) @@ -317,7 +322,7 @@ def verify_receipt(stage: Path, payload_path: Path, tag: str) -> dict: or receipt.get("base_sha256") != driver.BASE_SHA256 ): raise SystemExit( - "Strict export receipt does not name release 20261006 as its base " + "Strict export receipt does not name release 20261010 as its base " f"({receipt.get('base_tag')!r}, {receipt.get('base_sha256')!r})" ) if not receipt.get("files"): @@ -369,8 +374,8 @@ def verify_installed_exclusions( ) -> str: """The stage scores on the release's exclusion record, as installed. - ``text`` is release 20261006's record plus the ten ruled records - (finish_haiku55.build_release_exclusions, spelled by exclusions_text), + ``text`` is release 20261010's record plus the spec's ruled records, none + here (finish_musewonder.build_release_exclusions, spelled by exclusions_text), rebuilt now from git and the spec. The staged record must be those bytes, the receipt must bind them, and stage.json, which the receipt binds, must record them as --step install-exclusions wrote them: the installed and @@ -386,7 +391,7 @@ def verify_installed_exclusions( f"Staged {EXCLUSIONS} is not the installed engine upgrade's record" ) raise SystemExit( - f"Staged {EXCLUSIONS} is not release 20261006's record plus the " + f"Staged {EXCLUSIONS} is not release 20261010's record plus the " f"{driver.NEW_EXCLUSIONS} ruled records ({driver.SPEC_PATH}); run " "--step install-exclusions, then triage and export again" ) @@ -495,14 +500,14 @@ def verify_references( """References match the baseline or the gated installed upgrade's pins. The reference outputs, the scenarios and their sidecars must equal release - 20261006's pins in the stage, in the committed snapshot and in the + 20261010's pins in the stage, in the committed snapshot and in the manifest. The staged exclusion record must be the release's (``exclusions_sha256``). Its committed copy and its manifest pin are - release 20261006's before this freeze and the release's after it, so a + release 20261010's before this freeze and the release's after it, so a second freeze finds either. With an upgrade, the build's three reference files replace the staged pins, while committed copies and manifest pins may be baseline or build. - Scenarios and their sidecar remain pinned to release 20261006. + Scenarios and their sidecar remain pinned to release 20261010. """ pins = manifest["source_run_artifacts"][RUN]["files"] upgraded = {} if upgrade is None else driver.upgraded_reference_pins(upgrade) @@ -513,15 +518,15 @@ def matches(path: Path, allowed: set[str]) -> bool: for name, pin in driver.BASE_REFERENCE_SHA256.items(): staged, committed = {pin}, {pin} reason = "this release revises no reference value" - whose = "not release 20261006's" + whose = "not release 20261010's" if name == EXCLUSIONS: staged, committed = {exclusions_sha256}, {pin, exclusions_sha256} reason = "it is not the release's record" - whose = "neither release 20261006's nor this release's" + whose = "neither release 20261010's nor this release's" elif upgrade is not None and name in driver.UPGRADED_FILES: staged, committed = {upgraded[name]}, {pin, upgraded[name]} reason = "it is not the installed engine upgrade's file" - whose = "neither release 20261006's nor the installed engine upgrade's" + whose = "neither release 20261010's nor the installed engine upgrade's" if not matches(source_us / name, staged): raise SystemExit(f"Staged {name} changed: {reason}") if not matches(frozen_run / name, committed): @@ -536,7 +541,7 @@ def matches(path: Path, allowed: set[str]) -> bool: if not matches(frozen_run.parents[1] / "us_reference_outputs.csv", allowed): raise SystemExit( "Committed us_reference_outputs.csv changed: it is neither " - "release 20261006's nor the installed engine upgrade's" + "release 20261010's nor the installed engine upgrade's" ) if ( manifest["committed_snapshot_artifacts"].get("us_reference_outputs.csv") @@ -544,7 +549,7 @@ def matches(path: Path, allowed: set[str]) -> bool: ): raise SystemExit( "Manifest pin for us_reference_outputs.csv is neither release " - "20261006's nor the installed engine upgrade's" + "20261010's nor the installed engine upgrade's" ) if manifest["audit_annotation_artifacts"]["files"].get(EXPLANATIONS) not in { driver.BASE_EXPLANATIONS_SHA256, @@ -552,7 +557,7 @@ def matches(path: Path, allowed: set[str]) -> bool: }: raise SystemExit( f"Manifest pin for {EXPLANATIONS} is neither release " - "20261006's nor the installed engine upgrade's" + "20261010's nor the installed engine upgrade's" ) @@ -599,18 +604,18 @@ def verify_adjudication_record( cases_dir: Path, upgrade: driver.Upgrade | None = None, ) -> int: - """The staged record changes release 20261006's only as the release says. + """The staged record changes release 20261010's only as the release says. - The baseline is release 20261006's record in git (BASE_COMMIT), never the - working-tree copy this freeze overwrites. The ten ruled outputs must carry - exactly the entries the driver builds for them from the spec and each - case's bound verdict. A case Claude Haiku 5.5 re-opened may rewrite its + The baseline is release 20261010's record in git (BASE_COMMIT), never the + working-tree copy this freeze overwrites. Any ruled output must carry + exactly the entry the driver builds for it from the spec and its case's + bound verdict (there are none here). A case Muse Wonder re-opened may rewrite its judge fields, restated as the restate script does, and its reasoning exactly as the listed wording amendments say. Nothing else in any committed entry may change. The record excludes exactly the outputs the staged exclusion record lists. The staged file's bytes must be exactly its - parsed content, with release 20261006's note, and its date conventions - with the 2026-10-06 wave named (driver.release_date_conventions). Returns + parsed content, with release 20261010's note, and its date conventions + with any wave the release adds named (driver.release_date_conventions). Returns how many staged decisions, beyond the ruled ones, the committed record lacks. The staged record, prompt-changes.json and the amendments are bound by the receipt, which export writes only after triage applied them. @@ -719,18 +724,18 @@ def rebuild_payload( The receipt binds the payload only by a hash that sits beside it, so a payload edited after export (the addition's scores, an incumbent's cost, a case's classes) with its receipt rehashed passes every other gate. - Export's own build (driver.build_payload, against release 20261006 as + Export's own build (driver.build_payload, against release 20261010 as ``live``) runs again on a scratch copy of the bundle files the receipt binds, each checked against its receipt hash, and its bytes must equal the staged payload's. That build scores every model on the release's outputs - and repeats export's scope check: with release 20261006's exclusion + and repeats export's scope check: with release 20261010's exclusion record put back, every incumbent's modelStats is that release's, byte for byte. The copy is the bundle's: export_full_run writes data.json, us/data.json and us/analysis/ into the bundle it reads, and its data.json lacks the carried usage, so a rebuild in place would rewrite the stage. """ bundle = Path("publish") / RUN - with tempfile.TemporaryDirectory(prefix="freeze-haiku55-") as scratch: + with tempfile.TemporaryDirectory(prefix="freeze-musewonder-") as scratch: copy_root = Path(scratch) / RUN for name, expected in receipt["files"].items(): if not Path(name).is_relative_to(bundle): @@ -753,7 +758,7 @@ def rebuild_payload( def base_prediction_rows(): - """Release 20261006's predictions, read from git at BASE_COMMIT. + """Release 20261010's predictions, read from git at BASE_COMMIT. Never the working-tree copy, which the freezer rewrites from the stage: a freeze that stopped partway would compare the staged rows with themselves. @@ -769,7 +774,7 @@ def base_prediction_rows(): def base_serving_config() -> dict: - """Release 20261006's serving configuration, read from git at BASE_COMMIT.""" + """Release 20261010's serving configuration, read from git at BASE_COMMIT.""" return json.loads(driver.base_commit_blob(SNAPSHOT / "model_serving_config.json")) @@ -872,11 +877,11 @@ def amendments_text(record: dict) -> str: def committed_amendments(stage: Path) -> bytes: """The wording amendments record this freeze commits. - Release 20261006's committed record (from git at BASE_COMMIT) lists the + Release 20261010's committed record (from git at BASE_COMMIT) lists the amendments GPT-6.1 Sol's re-opened cases needed, and that wording is still published, so the record is never replaced. With no stage list, or an empty one, it is committed unchanged. A stage list (driver.load_amendments - checks each item names a case Claude Haiku 5.5 re-opened) is appended + checks each item names a case Muse Wonder re-opened) is appended after the earlier amendments, in its own order, and the note says which items are this release's; a note the stage list carries follows it. """ @@ -892,13 +897,13 @@ def committed_amendments(stage: Path) -> bytes: "amendments", ]: raise SystemExit( - f"Release 20261006's {COMMITTED_AMENDMENTS} is not in its committed " + f"Release 20261010's {COMMITTED_AMENDMENTS} is not in its committed " "form; cannot append this release's wording amendments" ) earlier = len(record["amendments"]) note = ( f"{record['note']} The last {len(added)} (after the first {earlier}) " - "are for cases Claude Haiku 5.5 re-opened (finish_haiku55.py " + "are for cases Muse Wonder re-opened (finish_musewonder.py " "load_amendments)." ) if isinstance(listed.get("note"), str) and listed["note"].strip(): @@ -943,7 +948,7 @@ def serving_problems( """Where a serving configuration is not the release's. Its models are the incumbents and the addition. Every incumbent's row is - release 20261006's (``previous``), every field. Each new model's evidence + release 20261010's (``previous``), every field. Each new model's evidence is its run state and names its supervised run. The evidence summary is the tally of the rows' evidence kinds. """ @@ -955,7 +960,7 @@ def serving_problems( if set(models) != expected: problems.append(f"roster differs on {sorted(set(models) ^ expected)[:8]}") problems += [ - f"{model} is not release 20261006's row" + f"{model} is not release 20261010's row" for model in sorted(set(incumbents) & set(models)) if json.dumps(models[model], sort_keys=True) != json.dumps(previous["models"][model], sort_keys=True) @@ -986,7 +991,7 @@ def merge_serving( ) -> dict: """The release's serving configuration, from the freezer's (``serving``). - The old rows already have frozen evidence. They are release 20261006's, + The old rows already have frozen evidence. They are release 20261010's, kept field for field after the incumbents' predictions were checked identical, instead of being silently downgraded when this checkout lacks the historical run directories. Each new model keeps the freezer's row, @@ -1032,19 +1037,35 @@ def release_pointer(tag: str, payload_path: Path, payload_hash: str) -> dict: def count_exclusions( text: str, exclusions: list[dict], upgrade: driver.Upgrade | None = None ) -> str: - """Count the record's engines, including the installed build's engine.""" - if upgrade is None: - return base_count_exclusions(text, exclusions) - engines = (*sorted(EXCLUSION_ENGINES), upgrade.engine_version) - version = upgrade.engine_version.removeprefix("policyengine-us ") + """The live version's description with the exclusions counted by engine. + + Release 20261010's description counts its records on 1.755.4, 2.15.17 and + 2.38.6 and names 2.38.6 as the reference engine. With an installed upgrade + the new engine is counted too (when it decided any record), named as the + reference engine, and the re-review clause counts the excluded outputs + that move on it. Applying this to its own result changes nothing. + """ + engines = list(BASE_EXCLUSION_ENGINES) + if upgrade is not None and upgrade.engine_version not in engines: + engines.append(upgrade.engine_version) + numbers = [engine.removeprefix("policyengine-us ") for engine in engines] counts = { engine: sum(r["engine_version"] == engine for r in exclusions) for engine in engines } + if sum(counts.values()) != len(exclusions): + raise SystemExit("an exclusion record names an engine the description omits") + fixed = len(BASE_EXCLUSION_ENGINES) pattern = ( - r"the \d+ excluded outputs keep the values they were decided on: \d+ from " - r"policyengine-us 1\.755\.4, \d+ from 2\.15\.17" - rf"(?:, \d+ from {re.escape(version)})?" + r"the \d+ excluded outputs keep the values they were decided on: " + + ", ".join( + (r"\d+ from policyengine-us " if i == 0 else r"\d+ from ") + + re.escape(number) + for i, number in enumerate(numbers[:fixed]) + ) + + "".join( + rf"(?:, \d+ from {re.escape(number)})?" for number in numbers[fixed:] + ) ) if len(re.findall(pattern, text)) != 1: raise SystemExit("the version description changed form") @@ -1052,13 +1073,16 @@ def count_exclusions( f"{counts[engine]} from " + (engine if i == 0 else engine.removeprefix("policyengine-us ")) for i, engine in enumerate(engines) - if i < len(EXCLUSION_ENGINES) or counts[engine] + if i < fixed or counts[engine] ] counted = ( f"the {len(exclusions)} excluded outputs keep the values they were decided on: " + ", ".join(described) ) - text = re.sub(pattern, counted, text) + text = re.sub(pattern, lambda _: counted, text) + if upgrade is None: + return text + version = upgrade.engine_version.removeprefix("policyengine-us ") previous = driver.BASE_ENGINE.removeprefix("policyengine-us ") versions = rf"(?:{re.escape(previous)}|{re.escape(version)})" text, changed = re.subn( @@ -1134,7 +1158,7 @@ def manifest_problems( A leaf may differ only under a MANIFEST_CHANGES path, and never at a MANIFEST_PINNED one: the pinned references and the reference explanations - keep release 20261006's hashes. + keep release 20261010's hashes. """ pinned = MANIFEST_PINNED changes = MANIFEST_CHANGES @@ -1214,13 +1238,13 @@ def verify_frozen( ) -> None: """What the freezer wrote is the release, checked file by file. - The frozen references are release 20261006's and the frozen exclusion + The frozen references are release 20261010's and the frozen exclusion record is the release's. The annotations directory holds the staged adjudications and annotation CSVs, byte for byte, and the wording amendments record; the frozen adjudication record is in the committed - form with the 2026-10-06 wave named. The serving configuration keeps - release 20261006's rows and adds the new model's. The manifest differs - from release 20261006's only where the release may change it, and states + form with any wave the release adds named. The serving configuration keeps + release 20261010's rows and adds the new model's. The manifest differs + from release 20261010's only where the release may change it, and states the release's snapshot date, model count, response window, exclusion counts, artifact and exclusion pin. """ @@ -1257,7 +1281,7 @@ def verify_frozen( ) if (annotations / COMMITTED_AMENDMENTS).read_bytes() != committed_amendments(stage): raise SystemExit( - f"The frozen {COMMITTED_AMENDMENTS} is not release 20261006's " + f"The frozen {COMMITTED_AMENDMENTS} is not release 20261010's " "record with the stage's list appended" ) serving = serving_problems( @@ -1422,7 +1446,7 @@ def main(argv: list[str] | None = None) -> None: exclusions_sha256 = verify_installed_exclusions( stage, receipt, exclusions_text, upgrade ) - # Claude Haiku 5.5's run files are the committed pins' and its bundle rows + # Muse Wonder's run files are the committed pins' and its bundle rows # are its run file's, every column: its cost, tokens and latency included. driver.verify_new_model_inputs(stage) runs = evidence_runs() @@ -1445,7 +1469,7 @@ def main(argv: list[str] | None = None) -> None: snapshot = ROOT / SNAPSHOT frozen_run = snapshot / "runs" / RUN - # The incumbents' frozen serving evidence is release 20261006's, from git. + # The incumbents' frozen serving evidence is release 20261010's, from git. previous_serving = base_serving_config() incumbents = set(previous_serving["models"]) - set(NEW_MODELS.values()) if len(incumbents) != driver.BASE_MODELS or model_names != incumbents | set( diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index 9c658442..d7d26978 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -1653,7 +1653,8 @@ def verify_fix_module_pins(fix_modules: list[dict], earlier: list[dict]) -> None def verify_regeneration_target_pins(upgrade: dict) -> None: """Each regenerated output held to the audit's fix modules publishes the bytes that ran: every module and sibling its target pins is the file - committed under AUDIT_FIXES_DIR, and its evidence file is the committed + committed under AUDIT_FIXES_DIR (or at the path it names, in an upgrade + audit's own fixes/), and its evidence file is the committed one. The builder and driver check this before the install; the freeze checks what the published sidecar says.""" for entry in upgrade.get("regenerated_exclusions", []): @@ -1665,7 +1666,18 @@ def verify_regeneration_target_pins(upgrade: dict) -> None: if not target.get("modules"): raise SystemExit(f"regenerated {key}: a fix_modules target pins no module") for item in pins: - path = f"{AUDIT_FIXES_DIR}/{item.get('module')}" + # A module an upgrade audit adds after 2026-09-22 names its own + # path, in that audit's fixes/ (the 2026-10-12 builder's + # fix_location); every other module is under AUDIT_FIXES_DIR. + path = item.get("path") or f"{AUDIT_FIXES_DIR}/{item.get('module')}" + if not re.fullmatch( + r"reference_audit/[0-9a-z-]+/fixes/" + re.escape(str(item.get("module"))), + path, + ): + raise SystemExit( + f"regenerated {key}: {item.get('module')} names path {path!r}, " + "not an audit's fixes/ directory" + ) committed = committed_sha256(path) if committed is None or item.get("sha256") != committed: raise SystemExit( From 0271aca7645d990a723d19181ee7006a619ae9ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 20:01:10 -0400 Subject: [PATCH 05/28] WIP: port the driver and builder tests to release 20261010's base tests/test_finish_musewonder.py ports the Haiku 5.5 driver's tests: - the ruled-exclusion sections become tests that the release rules nothing, so its record is release 20261010's; - the seed tests check the rebuilt 677-case seed against the two Haiku 5.5 records it derives from, and each rebuilt verdict against its transcript; - the upgrade tests regenerate release 20261010's own engine-defect records; - the date conventions name only a wave this release adds. tests/test_build_references_upgrade_1012.py tests the 2026-10-12 builder on a synthetic base with release 20261010's two earlier upgrades, its real base, and this audit's own fix module (r33). It loads its scripts under their own names, so the 2026-10-09 tests' builder never stands in for this one. Driver: triage decisions are dated by the engine upgrade's revision day, and that wave is named whenever the upgrade decides anything (DecisionWave): with no ruled wave, a flag on a regenerated reference could not be affirmed otherwise. The recovery script records each rebuilt verdict's requested judge model, the one its transcript's assistant turns name, which the restate script reads. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-12-engine-upgrade/upstreams.json | 15 + scripts/finish_musewonder.py | 83 +- scripts/freeze_musewonder.py | 6 +- scripts/recover_20261010_audit.py | 11 +- tests/test_build_references_upgrade_1012.py | 1783 +++++ tests/test_finish_musewonder.py | 6635 +++++++++++++++++ 6 files changed, 8507 insertions(+), 26 deletions(-) create mode 100644 reference_audit/2026-10-12-engine-upgrade/upstreams.json create mode 100644 tests/test_build_references_upgrade_1012.py create mode 100644 tests/test_finish_musewonder.py diff --git a/reference_audit/2026-10-12-engine-upgrade/upstreams.json b/reference_audit/2026-10-12-engine-upgrade/upstreams.json new file mode 100644 index 00000000..8a9b293f --- /dev/null +++ b/reference_audit/2026-10-12-engine-upgrade/upstreams.json @@ -0,0 +1,15 @@ +{ + "note": "The upstream policyengine-us fix each regeneration names (actions_from_cells.py --upstreams). An output's own entry wins over its root cause's; a record with several root causes names each one's fix. CA scenario_099's state income tax names #10052 too: its record's root causes (r02, r11) were fixed in 2.38.6, and the California educator-expense addback its 2026-10-10 recheck found (r33, this audit's fixes/) is #10052's. MA scenario_081's r22 stays excluded for its second reason, an unlisted input, so it needs none.", + "outputs": { + "scenario_099|state_income_tax_before_refundable_credits": "PolicyEngine/policyengine-us#10032; PolicyEngine/policyengine-us#10034; PolicyEngine/policyengine-us#10052" + }, + "root_causes": { + "r02_ira_219g": "PolicyEngine/policyengine-us#10032", + "r06_wi_act15_before_refundable": "PolicyEngine/policyengine-us#10026", + "r07_idaho_health_premiums": "PolicyEngine/policyengine-us#10023", + "r08_eitc_earned_income_deferrals": "PolicyEngine/policyengine-us#10030", + "r11_ca_itemized_conformity": "PolicyEngine/policyengine-us#10034", + "r30_snap_heat_and_eat_sua": "PolicyEngine/policyengine-us#10033", + "r32_wi_capital_gain_distributions": "PolicyEngine/policyengine-us#10026" + } +} diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py index 0618d1e5..22fc3925 100644 --- a/scripts/finish_musewonder.py +++ b/scripts/finish_musewonder.py @@ -1035,6 +1035,13 @@ def triage_items(upgrade, spec: dict | None = None) -> list[dict]: all(item["reference_verdict"] == "affirmed" for item in items), "a triage decision may only affirm a reference; an exclusion is ruled", ) + day = upgrade.revision["date"] + wrong = sorted(spec_key(i) for i in items if i["adjudicated_on"] != day) + require( + not wrong, + f"a triage decision is dated by the engine upgrade's wave ({day}), " + f"the one this release names: {wrong}", + ) return sorted(items, key=spec_key) @@ -1135,16 +1142,9 @@ def exclusion_adjudications( adjudicated_on=upgraded["decided_on"], ) ) - waves = {upgrade_wave(added)} - {None} - if spec_records(spec): - waves.add(spec["decided_on"]) + wave = decision_wave(added, triage) for item in triage: k = spec_key(item) - require( - item["adjudicated_on"] in waves, - f"{k}: a triage decision is dated by a wave this release names " - f"({sorted(waves)}), not {item['adjudicated_on']}", - ) require( k not in by_key, f"{k}: a triage decision on a case that already has a decision; " @@ -1152,23 +1152,48 @@ def exclusion_adjudications( ) appended.append(triage_entry(item, cases_dir / f"us__{k[0]}__{k[1]}")) out["adjudications"] = out["adjudications"] + appended - out["date_conventions"] = release_date_conventions( - out["date_conventions"], upgrade_wave(added) - ) + out["date_conventions"] = release_date_conventions(out["date_conventions"], wave) return out -def release_date_conventions(conventions: str, wave: str | None = None) -> str: +@dataclass(frozen=True) +class DecisionWave: + """The wave of an installed engine upgrade's decisions: its revision's UTC + day, and whether its decisions exclude outputs the upgrade moved onto an + unstated input (upgrade_decisions) or affirm references it changed + (triage_items).""" + + day: str + excludes: bool + affirms: bool + + +def decision_wave(added, triage) -> DecisionWave | None: + """The DecisionWave of the upgrade's new exclusions (``added``, as + upgrade_decisions pairs them) and triage decisions (``triage``), or None + when the upgrade decides nothing. Both are dated by one day: a new record + by its decided_on, a triage item by its adjudicated_on.""" + days = {record["decided_on"] for _, record in added} + days |= {item["adjudicated_on"] for item in triage} + require(len(days) <= 1, f"the upgrade's decisions name several days: {days}") + if not days: + return None + return DecisionWave(next(iter(days)), bool(added), bool(triage)) + + +def release_date_conventions( + conventions: str, wave: DecisionWave | None = None +) -> str: """Release 20261010's date conventions with each wave this release adds named; the one definition the adjudicate step writes and the record gate expects. A wave is added for the spec's ruled records, when it rules any (its decided_on, with its wave_sentence and adjudications_written_on), and for - ``wave``, the UTC day of an installed engine upgrade whose newly excluded - outputs are decided (upgrade_wave), with the day the spec says those - decisions were written (upgrade_adjudications_written_on). With neither, - the conventions are release 20261010's, unchanged. + ``wave``, the installed engine upgrade's decisions (decision_wave), with + the day the spec says those decisions were written + (upgrade_adjudications_written_on). With neither, the conventions are + release 20261010's, unchanged. """ require( conventions.count(DATE_CONVENTIONS_BASE) == 1 @@ -1191,12 +1216,24 @@ def release_date_conventions(conventions: str, wave: str | None = None) -> str: f"{SPEC_PATH} names no upgrade_adjudications_written_on, the UTC day " "the engine upgrade's decisions were written", ) - require(wave not in days, f"the {wave} wave is named twice") - days.append(wave) + require(wave.day not in days, f"the {wave.day} wave is named twice") + days.append(wave.day) + upgrade = f"the {wave.day} engine upgrade of the references" + what = [] + if wave.excludes: + what.append( + f"exclude the outputs {upgrade} moved onto an input the prompt " + "does not state" + ) + if wave.affirms: + what.append( + f"affirm references {upgrade} changed, which a re-judged case's " + "verdict flagged" + ) + require(bool(what), f"the {wave.day} wave decides nothing") sentence += ( - f" The {wave} wave's decisions exclude the outputs the references' " - "second engine upgrade moved onto an input the prompt does not state, " - f"and were written on {upgrade_written} UTC." + f" The {wave.day} wave's decisions {' and '.join(what)}, and were " + f"written on {upgrade_written} UTC." ) if not days: return conventions @@ -1334,7 +1371,7 @@ def record_text(record: dict) -> str: return json.dumps(record, indent=2, ensure_ascii=False) + "\n" -def verify_record_form(text: str, base: dict, wave: str | None = None) -> None: +def verify_record_form(text: str, base: dict, wave: DecisionWave | None = None) -> None: """The staged record's bytes are exactly its parsed content, and its top level other than the entries is release 20261010's. @@ -1747,7 +1784,7 @@ def stage_adjudications( base_record = base_adjudication_record() base = base_adjudications() current = path.read_text() - wave = upgrade_wave(added) + wave = decision_wave(added, triage) verify_record_form(current, base_record, wave) record = json.loads(current) grouped = _record_amendments(amendments) diff --git a/scripts/freeze_musewonder.py b/scripts/freeze_musewonder.py index 4d4f1162..14dc2159 100644 --- a/scripts/freeze_musewonder.py +++ b/scripts/freeze_musewonder.py @@ -630,7 +630,7 @@ def verify_adjudication_record( driver.verify_record_form( staged.read_text(), driver.base_adjudication_record(), - driver.upgrade_wave(decisions), + driver.decision_wave(decisions, driver.triage_items(upgrade)), ) after = load_adjudications(staged) base = driver.base_adjudications() @@ -1277,7 +1277,9 @@ def verify_frozen( driver.verify_record_form( (annotations / ADJUDICATIONS).read_text(), driver.base_adjudication_record(), - driver.upgrade_wave(driver.upgrade_decisions(upgrade)), + driver.decision_wave( + driver.upgrade_decisions(upgrade), driver.triage_items(upgrade) + ), ) if (annotations / COMMITTED_AMENDMENTS).read_bytes() != committed_amendments(stage): raise SystemExit( diff --git a/scripts/recover_20261010_audit.py b/scripts/recover_20261010_audit.py index 3921e494..0a5ea41d 100644 --- a/scripts/recover_20261010_audit.py +++ b/scripts/recover_20261010_audit.py @@ -54,12 +54,15 @@ def fail(msg): for path in sorted(glob.glob(str(cfg / "projects/*/*.jsonl"))): scanned += 1 texts, structured, refused, version, session = [], [], set(), set(), None + answered_by = set() for line in open(path, encoding="utf-8"): ev = json.loads(line) session = ev.get("sessionId", session) if ev.get("version"): version.add(ev["version"]) content = (ev.get("message") or {}).get("content") + if ev.get("type") == "assistant" and (ev.get("message") or {}).get("model"): + answered_by.add(ev["message"]["model"]) if ev.get("type") == "user" and isinstance(content, str): if not (ev.get("isMeta") is True and content == NUDGE): texts.append(content) @@ -82,7 +85,8 @@ def fail(msg): fail(f"two transcripts give {case} different verdict bytes") found.setdefault(case, {"prompt": prompt, "verdict": verdict, "transcript": path, "login": login, "session_id": session, - "cli_versions": sorted(version)}) + "cli_versions": sorted(version), + "answered_by": sorted(answered_by)}) print(f"scanned {scanned} transcripts; recovered {len(found)} of {len(want)}") missing = sorted(set(want) - set(found)) if missing: @@ -102,8 +106,13 @@ def fail(msg): (d / "verdict.json").write_bytes(item["verdict"]) shutil.copyfile(item["transcript"], d / "claude.transcript.jsonl") e = want[case] + if item["answered_by"] != e["judge_model_reported"]: + fail(f"{case}: the transcript's model {item['answered_by']} is not the provenance's") meta = { "judge_runner": "scripts/run_audit_claude.sh", + # The runner's AUDIT_MODEL (docs/haiku55/design.md), the one model the + # transcript's assistant turns name and the provenance reports. + "judge_model_requested": item["answered_by"][0], "verdict_sha256": e["verdict_sha256"], "prompt_sha256": e["prompt_sha256"], "judge_template_version": 1, diff --git a/tests/test_build_references_upgrade_1012.py b/tests/test_build_references_upgrade_1012.py new file mode 100644 index 00000000..39c6c4f0 --- /dev/null +++ b/tests/test_build_references_upgrade_1012.py @@ -0,0 +1,1783 @@ +"""The third engine upgrade's reference builder, its narratives and its draft +actions (reference_audit/2026-10-12-engine-upgrade, on release 20261010). + +Adapted from tests/test_build_references_upgrade.py, which tests the +2026-10-09 audit's copies. The planning logic is the same; the base, the +release record, the previous engine and this audit's own fix modules differ. + +The engine is stubbed: each test hands the builder's pure planning step a dict +of computed values. Properties (Hypothesis) state the builder's invariants: + +- every refusal fires: an unexplained move, scored or excluded; an approved + output that does not move; a regenerated output off its audited target; a new + exclusion on an output that does not move; a rechecked output that does not + move; a ruled output neither kept nor regenerated; +- a removed record's output is scored at its new engine value; +- every output the upgrade does not change keeps its bytes in the reference + CSV, and every earlier sidecar revision keeps its bytes; +- the records' order does not depend on the order of the actions' lists; +- the paper's partition of the revision's changes agrees with the builder. +""" + +from __future__ import annotations + +import copy +import importlib.util +import json +import random +import sys +import types +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +UPGRADE = ROOT / "reference_audit" / "2026-10-12-engine-upgrade" +SETTINGS = settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) + + +def _module(name: str, path: Path, builder=None): + """Load ``path`` as ``name``. This audit's narratives and drafting scripts + import ``build_references_upgrade`` by name, which the 2026-10-09 tests + may already have loaded from their folder: ``builder`` stands in under + that name while the module loads, and the earlier entry is put back.""" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + previous = sys.modules.get("build_references_upgrade") + if builder is not None: + sys.modules["build_references_upgrade"] = builder + try: + spec.loader.exec_module(module) + finally: + if builder is not None: + if previous is None: + sys.modules.pop("build_references_upgrade", None) + else: + sys.modules["build_references_upgrade"] = previous + return module + + +build = _module( + "build_references_upgrade_1012", + UPGRADE / "scripts" / "build_references_upgrade.py", +) +narratives = _module( + "narratives_upgrade_1012", UPGRADE / "scripts" / "narratives_upgrade.py", build +) +drafting = _module( + "actions_from_cells_1012", UPGRADE / "actions_from_cells.py", build +) + + +def test_this_audits_scripts_use_this_audits_builder(): + assert build.BASE_COMMIT == "5a8164a001efb27fa55f47fe7ea26666a0de31f8" + assert build.PREVIOUS_ENGINE == "policyengine-us 2.38.6" + assert narratives.BASE_COMMIT == build.BASE_COMMIT + assert drafting.PREVIOUS_ENGINE == build.PREVIOUS_ENGINE + assert drafting.load_base is build.load_base + +ENGINE = "policyengine-us 9.9.9" +DATE = "2026-10-09" +NOW = "2026-10-09T12:00:00+00:00" +S = "state_income_tax_before_refundable_credits" +F = "federal_income_tax_before_refundable_credits" +FLAG = "head_medicaid_eligible" + + +# --- A small synthetic base -------------------------------------------------- + + +def _defect(sid, variable, frozen, alternative, decided_on="2026-09-22"): + return { + "scenario_id": sid, + "variable": variable, + "reason_code": "reference_engine_defect", + "root_cause": f"r_{sid}", + "alternative_reading": "The rule as the law states it.", + "frozen_value": frozen, + "alternative_value": alternative, + "engine_version": "policyengine-us 1.755.4", + "decided_on": decided_on, + "decided_by": "developer", + "defect": "a defect", + "law": "a law", + "upstream": "to be filed", + "note": "a note", + } + + +def _unlisted(sid, variable, frozen, alternative, decided_on="2026-09-05"): + return { + "scenario_id": sid, + "variable": variable, + "reason_code": "reference_depends_on_unlisted_input", + "unlisted_input": "an input", + "alternative_reading": "Another reading.", + "frozen_value": frozen, + "alternative_value": alternative, + "engine_version": "policyengine-us 1.755.4", + "decided_on": decided_on, + "decided_by": "developer", + "note": "a note", + } + + +# scenario, variable, base value, impact weight +ROWS = [ + ("scenario_001", S, 1000.0, ""), + ("scenario_001", "local_income_tax", 0.0, ""), + ("scenario_002", S, 500.25, "1.5"), + ("scenario_002", F, 250.5, ""), + ("scenario_003", S, 1200.0, ""), + ("scenario_003", "snap", 3000.0, ""), + ("scenario_004", S, 700.0, ""), + ("scenario_004", F, 4100.0, ""), + ("scenario_005", S, 820.35, ""), + ("scenario_005", FLAG, 1.0, "0.0"), + ("scenario_006", "local_income_tax", 0.0, ""), + ("scenario_006", "snap", 120.0, ""), + ("scenario_023", FLAG, 1.0, ""), +] +BASE_RECORDS = [ + _defect("scenario_002", F, 250.5, 300.0), # fixed below: regenerated + _defect("scenario_003", "snap", 3000.0, 2500.0), # unfixed + _unlisted("scenario_006", "snap", 120.0, 0.0), +] +TAIL = _unlisted("scenario_023", FLAG, 1.0, 0.0, decided_on="2026-09-29") +RULED = [ + { + **_defect("scenario_004", S, 700.0, 690.0, decided_on="2026-10-06"), + "engine_version": "policyengine-us 2.38.6", + }, + { + **_unlisted("scenario_004", F, 4100.0, 5300.0, decided_on="2026-10-06"), + "engine_version": "policyengine-us 2.38.6", + }, + { + **_unlisted("scenario_005", S, 820.35, 820.26, decided_on="2026-10-06"), + "reason_code": "reference_law_published_after_freeze", + "root_cause": "later", + "published": "later", + "law": "a law", + "engine_version": "policyengine-us 2.38.6", + }, +] +FIRST_REVISION = { + "date": "2026-09-29", + "kind": "engine_upgrade", + "engine_version": "policyengine-us 2.15.17", + "previous_engine_version": "policyengine-us 1.755.4", + "fix_modules": [], + "excluded_outputs_rechecked": [], + "changed": [], +} +# Release 20261010's upgrade, the base's last revision: this audit's builder +# upgrades from its engine. +SECOND_REVISION = { + "date": "2026-10-10", + "kind": "engine_upgrade", + "engine_version": "policyengine-us 2.38.6", + "previous_engine_version": "policyengine-us 2.15.17", + "fix_modules": [], + "excluded_outputs_rechecked": [], + "regenerated_exclusions": [], + "changed": [], +} + + +def _csv_text(rows=ROWS) -> str: + lines = ["scenario_id,variable,value,impact_weight\n"] + lines += [f"{s},{v},{repr(float(x))},{w}\n" for s, v, x, w in rows] + return "".join(lines) + + +def _base() -> build.Base: + exclusions = { + "schema_version": 1, + "rule": "rule", + "derivation": "Derived.", + "exclusions": copy.deepcopy(BASE_RECORDS) + [copy.deepcopy(TAIL)], + } + return build.Base( + reference=build.parse_reference_csv(_csv_text()), + meta={ + "programs": [], + "policyengine_bundles": {"us": {}}, + "reference_csv_sha256": "x", + "regenerated_at_utc": "2026-09-29T15:04:45+00:00", + "revisions": [ + {"date": "2026-09-22", "kind": "convention", "changed": []}, + copy.deepcopy(FIRST_REVISION), + copy.deepcopy(SECOND_REVISION), + ], + }, + exclusions=exclusions, + scenarios_csv="", + frozen={(s, v): x - 1.0 for s, v, x, _ in ROWS}, + ) + + +def _release(base: build.Base) -> tuple[dict, set]: + doc = copy.deepcopy(base.exclusions) + doc["exclusions"] = ( + doc["exclusions"][:-1] + + sorted(copy.deepcopy(RULED), key=build.key_of) + + doc["exclusions"][-1:] + ) + doc["derivation"] += " Ruled ten." + return doc, {build.key_of(r) for r in RULED} + + +def _unchanged() -> dict: + return {(s, v): x for s, v, x, _ in ROWS} + + +def _new_exclusion(sid, value) -> dict: + record = _unlisted(sid, "local_income_tax", value, 0.0, decided_on=DATE) + record["engine_version"] = ENGINE + return record + + +def _actions(**lists) -> dict: + """Actions on the synthetic base; unless given, the kept list is every + ruled output the actions do not regenerate.""" + actions = { + "engine": ENGINE, + "previous_engine": "policyengine-us 2.38.6", + "date": DATE, + "release_spec_sha256": "spec", + "approved": [], + "regenerated_exclusions": [], + "new_exclusions": [], + "excluded_rechecked": [], + } + actions.update(lists) + if "kept_exclusions_from_release" not in lists: + regenerated = {build.key_of(r) for r in actions["regenerated_exclusions"]} + actions["kept_exclusions_from_release"] = [ + {"scenario_id": k[0], "variable": k[1]} + for k in sorted({build.key_of(r) for r in RULED} - regenerated) + ] + return actions + + +def _regenerate(sid, variable, alternative, tolerance=1.0): + return { + "scenario_id": sid, + "variable": variable, + "alternative_value": alternative, + "tolerance": tolerance, + "upstream": "PolicyEngine/policyengine-us#1", + "basis": "Fixed upstream.", + } + + +def _full(): + """A complete upgrade: one approval, two regenerations (one ruled), one new + exclusion, one recheck, a within-$1 move.""" + computed = _unchanged() + computed[("scenario_001", S)] = 1010.0 # approved + computed[("scenario_002", F)] = 300.4 # regenerated (20261006 record) + computed[("scenario_004", S)] = 690.2 # regenerated (ruled) + computed[("scenario_006", "local_income_tax")] = 670.55 # new exclusion + computed[("scenario_003", "snap")] = 2900.0 # rechecked: off target + computed[("scenario_002", S)] = 500.75 # within $1 + actions = _actions( + approved=[ + { + "scenario_id": "scenario_001", + "variable": S, + "value": 1010.0, + "cause": "a_cause", + "basis": "A basis.", + } + ], + regenerated_exclusions=[ + _regenerate("scenario_002", F, 300.0), + _regenerate("scenario_004", S, 690.0), + ], + new_exclusions=[_new_exclusion("scenario_006", 670.55)], + excluded_rechecked=[ + {"scenario_id": "scenario_003", "variable": "snap", "reason": "Not fixed."} + ], + ) + return computed, actions + + +def _plan(computed, actions): + base = _base() + release, ruled = _release(base) + return base, release, build.plan_upgrade(base, release, ruled, actions, computed) + + +def _write(tmp_path, computed, actions): + base, release, plan = _plan(computed, actions) + assert plan.problems == [] + written = build.write_outputs( + tmp_path, + base, + release, + plan, + actions, + bundle={"policyengine_version": "6.1.2", "bundled_model_version": "2.2.1"}, + fix_modules=[{"module": "m.py", "sha256": "0"}], + provenance={"builder_sha256": "0"}, + traces={ + f"{c['scenario_id']}|{c['variable']}": { + "pe_variable": c["variable"], + "trace": "t\nu", + } + for c in plan.changed + plan.within + }, + now=NOW, + ) + return base, plan, written + + +# --- The full upgrade --------------------------------------------------------- + + +def test_a_complete_upgrade_writes_consistent_records(tmp_path): + computed, actions = _full() + base, plan, written = _write(tmp_path, computed, actions) + revision = written.meta["revisions"][-1] + assert revision["kind"] == "engine_upgrade" + assert revision["engine_version"] == ENGINE + assert revision["previous_engine_version"] == "policyengine-us 2.38.6" + assert written.meta["revisions"][:-1] == base.meta["revisions"] + assert [r["kind"] for r in written.meta["revisions"]].count("engine_upgrade") == 3 + causes = { + (c["scenario_id"], c["variable"]): c["cause"] for c in revision["changed"] + } + assert causes == { + ("scenario_001", S): "a_cause", + ("scenario_002", F): "regenerated_upstream_fix", + ("scenario_004", S): "regenerated_upstream_fix", + ("scenario_006", "local_income_tax"): ( + "excluded_reference_depends_on_unlisted_input" + ), + ("scenario_002", S): "engine_upgrade_within_1", + } + (rechecked,) = revision["excluded_outputs_rechecked"] + assert rechecked == { + "scenario_id": "scenario_003", + "variable": "snap", + "kept_value": 3000.0, + "value_on_9_9_9": 2900.0, + "reason": "Not fixed.", + } + assert [build.key_of(r) for r in revision["regenerated_exclusions"]] == [ + ("scenario_002", F), + ("scenario_004", S), + ] + assert revision["kept_exclusions_from_release"] == [ + {"scenario_id": "scenario_004", "variable": F}, + {"scenario_id": "scenario_005", "variable": S}, + ] + keys = [build.key_of(r) for r in written.exclusions["exclusions"]] + assert keys == [ + ("scenario_003", "snap"), + ("scenario_006", "snap"), + ("scenario_004", F), + ("scenario_005", S), + ("scenario_006", "local_income_tax"), + ("scenario_023", FLAG), + ] + assert written.exclusions["derivation"].startswith( + "Derived. Ruled ten. On 2026-10-09" + ) + derivation = written.exclusions["derivation"] + assert "from policyengine-us 2.38.6 to policyengine-us 9.9.9" in derivation + assert "behind two excluded outputs" in derivation + assert "One scored output the new engine moved was excluded" in derivation + assert "that stay excluded, one moved" in derivation + values = build.parse_reference_csv(written.csv_text).values() + assert values[("scenario_003", "snap")] == 3000.0 # rule 5: kept + assert values[("scenario_006", "local_income_tax")] == 670.55 + assert set(written.traces) == {f"{k[0]}|{k[1]}" for k in causes} + on_disk = (tmp_path / "reference_outputs.csv.meta.json").read_text() + assert on_disk == build.dump_record(written.meta) + assert ( + build.sha256_bytes(written.csv_text.encode()) + == (written.meta["reference_csv_sha256"]) + ) + + +def test_the_papers_partition_agrees_with_the_builder(tmp_path): + """Differential: the paper reads the written sidecar and record as the + builder planned them (engine_upgrades_from rebuilds the record around each + upgrade from regenerated_exclusions' removed records).""" + from policybench.paper_results import engine_upgrades_from + + computed, actions = _full() + _, plan, written = _write(tmp_path, computed, actions) + *_, upgrade = engine_upgrades_from( + written.meta["revisions"], written.exclusions["exclusions"] + ) + assert (upgrade.previous_engine_version, upgrade.engine_version) == ( + "2.38.6", + "9.9.9", + ) + groups = { + name: sorted(build.key_of(c) for c in group) + for name, group in upgrade.partition.items() + } + assert groups == { + "scored_changes": [("scenario_001", S)], + "within_tolerance": [("scenario_002", S)], + "new_exclusions": [("scenario_006", "local_income_tax")], + "restored": [("scenario_002", F), ("scenario_004", S)], + } + assert upgrade.restored == {build.key_of(r) for r in plan.regenerated} + assert [upgrade.rechecked_value(r) for r in upgrade.rechecked] == [2900.0] + + +def test_a_kept_ruled_output_that_moves_is_rechecked(tmp_path): + """A ruled output the release keeps (later law, here) that the new engine + moves is listed both as kept and as rechecked, and keeps its value (rule + 5). The draft lists it in both places, and the builder accepts it.""" + computed = _unchanged() + computed[("scenario_005", S)] = 830.0 + base = _base() + release, ruled = _release(base) + drafted, _ = drafting.draft_actions( + _cells(computed), + release, + ruled, + {build.key_of(r) for r in base.exclusions["exclusions"]}, + {}, + engine_version="9.9.9", + date=DATE, + spec_sha256="spec", + precise=True, + ) + actions = {k: v for k, v in drafted.items() if k not in ("draft", "review")} + assert ("scenario_005", S) in { + build.key_of(e) for e in actions["kept_exclusions_from_release"] + } & {build.key_of(e) for e in actions["excluded_rechecked"]} + build.validate_actions(actions) + _, plan, written = _write(tmp_path, computed, actions) + assert [build.key_of(r) for r in plan.rechecked] == [("scenario_005", S)] + assert ( + build.parse_reference_csv(written.csv_text).values()[("scenario_005", S)] + == 820.35 + ) + # Kept may pair with rechecked only. + for name in ("approved", "regenerated_exclusions", "new_exclusions"): + with pytest.raises(build.Refusal, match="is in both"): + build.validate_actions( + {**actions, name: [{"scenario_id": "scenario_005", "variable": S}]} + ) + + +def test_the_written_date_must_be_the_actions_date(tmp_path): + computed, actions = _full() + base, release, plan = _plan(computed, actions) + with pytest.raises(build.Refusal, match="does not fall on the actions date"): + build.write_outputs( + tmp_path, + base, + release, + plan, + actions, + bundle={}, + fix_modules=[], + provenance={}, + traces={}, + now="2026-10-10T00:00:01+00:00", + ) + + +# --- Refusals ------------------------------------------------------------------ + + +def _problems(computed, actions) -> list[str]: + return _plan(computed, actions)[2].problems + + +@SETTINGS +@given(delta=st.floats(min_value=1.01, max_value=1e5) | st.floats(-1e5, -1.01)) +def test_an_unexplained_scored_move_is_refused(delta): + computed = _unchanged() + computed[("scenario_001", S)] += delta + (problem,) = _problems(computed, _actions()) + assert problem.startswith("unreviewed move ('scenario_001'") + + +def test_a_flag_flip_is_a_move(): + computed = _unchanged() + computed[("scenario_005", FLAG)] = 0.0 + (problem,) = _problems(computed, _actions()) + assert "unreviewed move ('scenario_005', 'head_medicaid_eligible')" in problem + + +@SETTINGS +@given(delta=st.floats(min_value=1.01, max_value=1e4)) +def test_an_excluded_move_without_a_recheck_is_refused(delta): + computed = _unchanged() + computed[("scenario_006", "snap")] += delta + (problem,) = _problems(computed, _actions()) + assert problem.startswith("excluded output moved without a recheck") + + +@SETTINGS +@given(delta=st.floats(min_value=-1.0, max_value=1.0)) +def test_an_approved_output_that_does_not_move_is_refused(delta): + computed = _unchanged() + computed[("scenario_001", S)] += delta + approved = [ + { + "scenario_id": "scenario_001", + "variable": S, + "value": computed[("scenario_001", S)], + "cause": "c", + "basis": "b", + } + ] + (problem,) = _problems(computed, _actions(approved=approved)) + assert problem.startswith("approved ('scenario_001'") and "does not move" in problem + + +def test_an_approved_output_that_lands_elsewhere_is_refused(): + computed = _unchanged() + computed[("scenario_001", S)] = 1010.0 + approved = [ + { + "scenario_id": "scenario_001", + "variable": S, + "value": 1010.01, + "cause": "c", + "basis": "b", + } + ] + (problem,) = _problems(computed, _actions(approved=approved)) + assert "!= approved" in problem + + +@SETTINGS +@given( + miss=st.floats(min_value=0.0, max_value=50.0), + tolerance=st.floats(min_value=0.0, max_value=1.0), +) +def test_a_regenerated_output_must_land_within_its_tolerance(miss, tolerance): + computed = _unchanged() + computed[("scenario_002", F)] = 300.0 + miss + actions = _actions( + regenerated_exclusions=[_regenerate("scenario_002", F, 300.0, tolerance)] + ) + problems = _problems(computed, actions) + if abs(computed[("scenario_002", F)] - 300.0) <= tolerance: + assert problems == [] + else: + (problem,) = problems + assert "misses its audited target" in problem + + +def test_a_regeneration_must_name_the_records_target_and_a_small_tolerance(): + computed = _unchanged() + computed[("scenario_002", F)] = 300.0 + wrong_target = _actions( + regenerated_exclusions=[_regenerate("scenario_002", F, 301.0)] + ) + assert "is not the record's" in _problems(computed, wrong_target)[0] + wide = _actions(regenerated_exclusions=[_regenerate("scenario_002", F, 300.0, 1.5)]) + assert "tolerance 1.5 outside" in _problems(computed, wide)[0] + + +def test_only_an_engine_defect_is_regenerated(): + computed = _unchanged() + computed[("scenario_006", "snap")] = 0.0 + actions = _actions( + regenerated_exclusions=[_regenerate("scenario_006", "snap", 0.0)] + ) + (problem,) = _problems(computed, actions) + assert "not an engine defect" in problem + + +# --- A fix-modules target: a record decided on an older engine ---------------- +# +# scenario_003 snap's record was decided on 1.755.4 (frozen 3000, corrected +# 2500). Other engine changes have since moved the output, so its +# alternative_value is stale: the audited fix module, run on a pre-fix engine, +# gives the target instead. + +PRE_FIX = "policyengine-us 9.9.8" +MODULE = "r_fix_v2.py" +EVIDENCE = {"path": "evidence/pre_fix.json", "sha256": "e" * 64} + + +def _evidence(engine_value=3100.0, corrected=2600.3, engine=PRE_FIX, pin="p" * 64): + return { + "kind": build.EVIDENCE_KIND, + "engine": engine, + "modules": {MODULE: pin}, + "items": [ + { + "scenario_id": "scenario_003", + "variable": "snap", + "modules": [MODULE], + "engine_value": engine_value, + "corrected_value": corrected, + } + ], + } + + +def _module_target(**target): + entry = _regenerate("scenario_003", "snap", 2500.0) + entry["target"] = { + "kind": "fix_modules", + "modules": [MODULE], + "evidence": dict(EVIDENCE), + **target, + } + return entry + + +def _module_plan( + value=2600.0, after=2600.1, evidence=None, pins=None, entry=None, deps=None +): + computed = _unchanged() + computed[("scenario_003", "snap")] = value + base = _base() + release, ruled = _release(base) + actions = _actions(regenerated_exclusions=[entry or _module_target()]) + return build.plan_upgrade( + base, + release, + ruled, + actions, + computed, + evidence={EVIDENCE["path"]: evidence or _evidence()}, + module_pins={MODULE: "p" * 64} if pins is None else pins, + module_values={(("scenario_003", "snap"), (MODULE,)): after}, + module_deps=deps, + ) + + +def test_a_fix_modules_target_pins_the_siblings_its_modules_load(): + """r02_ira_219g_v2.py runs r02_ira_219g.py: the evidence must pin that + sibling at the bytes committed at the base, and the target records it.""" + sibling = "r_fix.py" + deps = {MODULE: [sibling]} + missing = _module_plan(deps=deps, pins={MODULE: "p" * 64, sibling: "s" * 64}) + assert any("does not pin the siblings" in p for p in missing.problems) + doc = _evidence() + doc["modules"][sibling] = "s" * 64 + plan = _module_plan( + deps=deps, evidence=doc, pins={MODULE: "p" * 64, sibling: "s" * 64} + ) + assert plan.problems == [] + (regenerated,) = plan.regenerated + assert regenerated["target"]["dependencies"] == [ + {"module": sibling, "sha256": "s" * 64} + ] + stale = _module_plan( + deps=deps, evidence=doc, pins={MODULE: "p" * 64, sibling: "t" * 64} + ) + assert any("does not pin the siblings" in p for p in stale.problems) + + +def test_module_dependencies_are_found_transitively(): + files = { + "a_v2.py": b"x = Path(__file__).with_name('a.py')", + "a.py": b'y = Path(__file__).with_name( "b.json" )', + "b.json": b"{}", + } + assert build.module_dependencies("a_v2.py", files.__getitem__) == ["a.py", "b.json"] + assert build.module_dependencies("b.json", files.__getitem__) == [] + + +def test_a_module_whose_closure_cannot_be_established_is_refused(): + """The delta review's case: c13v3_upstream_plus_r30.py loads six siblings + through _HERE / f"{name}.py"; pinning only the wrapper would let a loaded + sibling change unseen, so the builder refuses it.""" + with pytest.raises(build.Refusal, match="cannot be established"): + build.module_dependencies("c13v3_upstream_plus_r30.py", build.committed_fix) + with pytest.raises(build.Refusal, match="cannot be established"): + build.audit_module_pins(["c13v3_upstream_plus_r30.py"]) + + +def test_the_committed_modules_siblings_are_the_ones_they_load(): + """On the real fixes: r01, r02 and r03's v2 modules each load a sibling.""" + deps = { + m: build.module_dependencies(m, build.committed_fix) + for m in ( + "r01_ira_compensation_v2.py", + "r02_ira_219g_v2.py", + "r03_estate_income__qbi_false_v2.py", + "r07_idaho_health_premiums_v2.py", + ) + } + assert deps["r01_ira_compensation_v2.py"] == ["r01_ira_compensation.py"] + assert deps["r02_ira_219g_v2.py"] == ["r02_ira_219g.py"] + assert deps["r03_estate_income__qbi_false_v2.py"][0] == "r03_estate_income_v2.py" + assert deps["r07_idaho_health_premiums_v2.py"] == [] + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"after": float("nan")}, "still moves"), + ({"evidence": _evidence(corrected=float("nan"))}, "not finite"), + ({"value": float("nan"), "after": 2600.0}, "non-finite"), + ], +) +def test_a_non_finite_value_is_refused(kwargs, message): + problems = _module_plan(**kwargs).problems + assert any(message in p for p in problems), problems + + +def test_a_record_target_misses_a_drifted_output(): + computed = _unchanged() + computed[("scenario_003", "snap")] = 2600.0 + actions = _actions( + regenerated_exclusions=[_regenerate("scenario_003", "snap", 2500.0)] + ) + (problem,) = _problems(computed, actions) + assert "misses its audited target" in problem and "(record)" in problem + + +def test_a_fix_modules_target_regenerates_a_drifted_output(tmp_path): + plan = _module_plan() + assert plan.problems == [] + (regenerated,) = plan.regenerated + target = regenerated["target"] + assert target == { + "kind": "fix_modules", + "value": 2600.3, + "engine": PRE_FIX, + "engine_value": 3100.0, + "modules": [{"module": MODULE, "sha256": "p" * 64}], + "dependencies": [], + "evidence": EVIDENCE["path"], + "evidence_sha256": EVIDENCE["sha256"], + "value_with_modules": 2600.1, + } + assert regenerated["audited_alternative_value"] == 2500.0 + assert regenerated["regenerated"] == 2600.0 + assert plan.new_values[("scenario_003", "snap")] == 2600.0 + + +def test_a_record_target_is_the_default_and_names_its_engine(): + computed = _unchanged() + computed[("scenario_002", F)] = 300.4 + actions = _actions(regenerated_exclusions=[_regenerate("scenario_002", F, 300.0)]) + (regenerated,) = _plan(computed, actions)[2].regenerated + assert regenerated["target"] == { + "kind": "record", + "value": 300.0, + "engine": "policyengine-us 1.755.4", + } + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"evidence": _evidence(engine=ENGINE)}, "it must come from one"), + ( + {"evidence": _evidence(engine="policyengine-us 10.0.0")}, + "not an engine older", + ), + ({"pins": {MODULE: "q" * 64}}, "ran other bytes"), + ({"pins": {}}, "unknown fix modules"), + ({"evidence": _evidence(corrected=3100.5)}, "shows no defect"), + ({"after": 2590.0}, "still moves it"), + ({"value": 2610.0, "after": 2610.0}, "misses its audited target"), + ( + {"entry": _module_target(modules=["other.py"]), "pins": {"other.py": "0"}}, + "has 0 items", + ), + ({"entry": _module_target(modules=[])}, "names no modules"), + ({"entry": _module_target(extra=1)}, "takes kind, modules and evidence"), + ( + {"entry": _module_target(evidence={"path": "x", "sha256": "0"})}, + "evidence is not loaded", + ), + ({"entry": _module_target(kind="guess")}, "unknown target kind"), + ], +) +def test_a_fix_modules_target_refuses(kwargs, message): + problems = _module_plan(**kwargs).problems + assert problems and any(message in p for p in problems), problems + + +def test_a_record_target_takes_no_other_fields(): + computed = _unchanged() + computed[("scenario_002", F)] = 300.0 + entry = _regenerate("scenario_002", F, 300.0) + entry["target"] = {"kind": "record", "modules": [MODULE]} + (problem,) = _problems(computed, _actions(regenerated_exclusions=[entry])) + assert "takes no other fields" in problem + + +@SETTINGS +@given( + effect=st.floats(min_value=1.01, max_value=5e3) | st.floats(-5e3, -1.01), + drift=st.floats(min_value=-5e3, max_value=5e3), + landing=st.floats(min_value=-3.0, max_value=3.0), + leftover=st.floats(min_value=-3.0, max_value=3.0), +) +def test_a_fix_modules_regeneration_needs_a_landing_and_nothing_left_to_fix( + effect, drift, landing, leftover +): + """On the pre-fix engine the module moves the output by ``effect``; the new + engine lands ``landing`` from the corrected value, and the module, applied + again there, moves it by ``leftover``. It regenerates exactly when both + are within $1.""" + before = 3000.0 + drift + corrected = before + effect + value = corrected + landing + plan = _module_plan( + value=value, + after=value + leftover, + evidence=_evidence(engine_value=before, corrected=corrected), + ) + ok = abs(landing) <= 1.0 and abs(leftover) <= 1.0 + assert (plan.problems == []) == ok, plan.problems + if ok: + assert plan.new_values.get(("scenario_003", "snap"), 3000.0) in (value, 3000.0) + assert [r["target"]["value"] for r in plan.regenerated] == [corrected] + + +def test_the_derivation_names_the_audited_corrected_value(tmp_path): + computed, actions = _full() + _, _, written = _write(tmp_path, computed, actions) + derivation = written.exclusions["derivation"] + assert "within $1 of its audited corrected value" in derivation or ( + "each landing within $1 of its audited corrected value" in derivation + ) + assert "record's corrected value" not in derivation + + +@SETTINGS +@given(delta=st.floats(min_value=-1.0, max_value=1.0)) +def test_a_new_exclusion_on_an_output_that_does_not_move_is_refused(delta): + computed = _unchanged() + computed[("scenario_006", "local_income_tax")] = delta + actions = _actions(new_exclusions=[_new_exclusion("scenario_006", delta)]) + problems = _problems(computed, actions) + assert any("does not move" in p for p in problems) + + +def test_a_new_exclusion_must_carry_the_engine_value_and_date(): + computed = _unchanged() + computed[("scenario_006", "local_income_tax")] = 670.55 + record = _new_exclusion("scenario_006", 670.0) + record["decided_on"] = "2026-10-08" + problems = _problems(computed, _actions(new_exclusions=[record])) + assert len(problems) == 2 + assert "frozen_value 670.0 != engine 670.55" in problems[0] + assert "decided_on '2026-10-08' != '2026-10-09'" in problems[1] + + +def test_a_rechecked_output_that_does_not_move_is_refused(): + actions = _actions( + excluded_rechecked=[ + {"scenario_id": "scenario_006", "variable": "snap", "reason": "r"} + ] + ) + (problem,) = _problems(_unchanged(), actions) + assert problem.startswith("rechecked ('scenario_006', 'snap') does not move") + + +def test_every_ruled_output_is_kept_or_regenerated(): + actions = _actions( + kept_exclusions_from_release=[{"scenario_id": "scenario_004", "variable": F}] + ) + (problem,) = _problems(_unchanged(), actions) + assert f"missing [('scenario_004', '{S}'), ('scenario_005', '{S}')]" in problem + + +def test_actions_on_the_wrong_kind_of_output_are_refused(): + computed = _unchanged() + computed[("scenario_003", "snap")] = 0.0 + approved = [ + { + "scenario_id": "scenario_003", + "variable": "snap", + "value": 0.0, + "cause": "c", + "basis": "b", + } + ] + problems = _problems(computed, _actions(approved=approved)) + assert any( + "approved names ('scenario_003', 'snap'), which is excluded" in p + for p in problems + ) + rechecked = [{"scenario_id": "scenario_001", "variable": S, "reason": "r"}] + problems = _problems(_unchanged(), _actions(excluded_rechecked=rechecked)) + assert problems == [ + f"excluded_rechecked names ('scenario_001', '{S}'), which is scored" + ] + + +def test_the_actions_file_is_validated(): + with pytest.raises(build.Refusal, match="draft"): + build.validate_actions({**_actions(), "draft": True}) + build.validate_actions({**_actions(), "draft": True}, allow_draft=True) + with pytest.raises(build.Refusal, match="unknown actions keys"): + build.validate_actions({**_actions(), "approve": []}) + twice = _actions( + approved=[{"scenario_id": "scenario_001", "variable": S}], + excluded_rechecked=[{"scenario_id": "scenario_001", "variable": S}], + ) + with pytest.raises( + build.Refusal, match="is in both approved and excluded_rechecked" + ): + build.validate_actions(twice) + with pytest.raises(build.Refusal, match="policyengine-us X.Y.Z"): + build.validate_actions({**_actions(), "engine": "2.36.0"}) + + +def test_a_previous_engine_other_than_the_base_is_refused(): + problems = _problems( + _unchanged(), {**_actions(), "previous_engine": "policyengine-us 2.2.1"} + ) + assert problems == [ + "previous_engine is policyengine-us 2.2.1, not policyengine-us 2.38.6" + ] + + +# --- Properties -------------------------------------------------------------- + + +@SETTINGS +@given( + federal=st.floats(min_value=299.0, max_value=301.0), + ruled=st.floats(min_value=689.0, max_value=691.0), +) +def test_a_removed_record_is_scored_at_its_new_value(tmp_path, federal, ruled): + computed = _unchanged() + computed[("scenario_002", F)] = federal + computed[("scenario_004", S)] = ruled + actions = _actions( + regenerated_exclusions=[ + _regenerate("scenario_002", F, 300.0), + _regenerate("scenario_004", S, 690.0), + ] + ) + _, plan, written = _write(tmp_path, computed, actions) + values = build.parse_reference_csv(written.csv_text).values() + excluded = {build.key_of(r) for r in written.exclusions["exclusions"]} + for key, value in ((("scenario_002", F), federal), (("scenario_004", S), ruled)): + assert key not in excluded + assert values[key] == value + (change,) = [c for c in plan.changed if build.key_of(c) == key] + assert ( + change["regenerated"] == value + and change["cause"] == "regenerated_upstream_fix" + ) + (removed,) = [ + r + for r in written.meta["revisions"][-1]["regenerated_exclusions"] + if build.key_of(r) == ("scenario_002", F) + ] + assert removed["record"] == BASE_RECORDS[0] + assert removed["value_on_9_9_9"] == federal + + +@SETTINGS +@given( + moves=st.dictionaries( + st.sampled_from([(s, v) for s, v, _, _ in ROWS]), + st.floats(min_value=-1.0, max_value=1.0), + ) +) +def test_unchanged_outputs_keep_their_bytes(tmp_path, moves): + """Moves within the tolerance, anywhere: excluded outputs keep their values, + scored ones take the engine's and are listed; every other row keeps its + bytes; earlier revisions keep theirs.""" + computed = _unchanged() + for key, delta in moves.items(): + if key[1] == FLAG: + continue + computed[key] += delta + base, plan, written = _write(tmp_path, computed, _actions()) + excluded = {build.key_of(r) for r in _release(base)[0]["exclusions"]} + rows = build.parse_reference_csv(written.csv_text).rows + listed = {build.key_of(c) for c in plan.within} + for old, new in zip(base.reference.rows, rows, strict=True): + if old.key in excluded or abs(computed[old.key] - old.value) <= build.EPS: + assert new.raw == old.raw + assert old.key not in listed + else: + assert new.value == computed[old.key] + assert old.key in listed + for before, after in zip(base.meta["revisions"], written.meta["revisions"]): + assert json.dumps(before) == json.dumps(after) + + +@SETTINGS +@given(seed=st.integers(min_value=0, max_value=2**32 - 1)) +def test_the_records_do_not_depend_on_the_actions_order(tmp_path, seed): + computed, actions = _full() + shuffled = copy.deepcopy(actions) + rng = random.Random(seed) + for name in ( + "approved", + "regenerated_exclusions", + "new_exclusions", + "excluded_rechecked", + "kept_exclusions_from_release", + ): + rng.shuffle(shuffled[name]) + shuffled["new_exclusions"].append(_new_exclusion("scenario_001", 50.0)) + computed = {**computed, ("scenario_001", "local_income_tax"): 50.0} + actions = { + **actions, + "new_exclusions": actions["new_exclusions"] + + [_new_exclusion("scenario_001", 50.0)], + } + rng.shuffle(shuffled["new_exclusions"]) + _, _, first = _write(tmp_path / "a", computed, actions) + _, _, second = _write(tmp_path / "b", computed, shuffled) + for name in ( + "reference_exclusions.json", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + ): + assert (tmp_path / "a" / name).read_bytes() == ( + tmp_path / "b" / name + ).read_bytes() + keys = [build.key_of(r) for r in first.exclusions["exclusions"]] + assert keys[-3:] == [ + ("scenario_001", "local_income_tax"), + ("scenario_006", "local_income_tax"), + ("scenario_023", FLAG), + ] + + +@SETTINGS +@given( + values=st.dictionaries( + st.sampled_from([(s, v) for s, v, _, _ in ROWS]), + st.floats(allow_nan=False, allow_infinity=False, width=32), + ) +) +def test_rendering_rewrites_only_the_named_rows(values): + reference = build.parse_reference_csv(_csv_text()) + text = build.render_reference_csv(reference, values) + rows = build.parse_reference_csv(text).rows + for old, new in zip(reference.rows, rows, strict=True): + if old.key in values: + assert new.value == float(values[old.key]) + assert new.fields[3] == old.fields[3] + else: + assert new.raw == old.raw + + +# --- The real base --------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def real_base(): + return build.load_base() + + +def test_the_base_is_release_20261010(real_base): + assert len(real_base.reference.rows) == 1984 + assert len(real_base.exclusions["exclusions"]) == 58 + kinds = [r["kind"] for r in real_base.meta["revisions"]] + assert kinds.count("engine_upgrade") == 2 and kinds[-1] == "engine_upgrade" + assert real_base.meta["revisions"][-1]["engine_version"] == build.PREVIOUS_ENGINE + assert build.key_of(real_base.exclusions["exclusions"][-1]) == build.AUDIT_TAIL + # The base CSV round-trips byte for byte through the row parser. + raw = build.git_blob(build.BASE_COMMIT, f"{build.RUN}/reference_outputs.csv") + assert build.render_reference_csv(real_base.reference, {}) == raw.decode() + + +def test_the_release_record_is_the_drivers(real_base): + """Differential: the record the builder edits is finish_musewonder's, + release 20261010's own, since the release rules nothing.""" + release, ruled, spec_sha = build.release_record(real_base.exclusions) + driver = sys.modules["finish_musewonder_for_upgrade"] + expected = driver.build_release_exclusions( + driver.base_exclusion_record(), driver.load_spec() + ) + assert release == expected == real_base.exclusions + assert len(release["exclusions"]) == 58 and ruled == set() + assert spec_sha == build.sha256(ROOT / "docs" / "musewonder" / "spec.json") + assert build.key_of(release["exclusions"][-1]) == build.AUDIT_TAIL + + +def test_the_fix_modules_are_the_first_upgrades(real_base, tmp_path): + """In place, the conventions are the 2026-09-29 upgrade's; the sales tax + tables must sit beside them.""" + import shutil + + for path in build.FIXES.glob("*.py"): + shutil.copy(path, tmp_path / path.name) + with pytest.raises(build.Refusal, match="r19_irs_sales_tax_2025.json is missing"): + build.fix_module_pins(real_base.meta, tmp_path) + shutil.copy(build.SALES_TAX_SOURCE, tmp_path / build.SALES_TAX_TABLES) + pins = build.fix_module_pins(real_base.meta, tmp_path) + assert [p["module"] for p in pins][-2:] == [ + "latest_final.py", + build.SALES_TAX_TABLES, + ] + assert len(pins) == 13 + (tmp_path / "latest_c_md_2026.py").write_text("# edited\n") + with pytest.raises(build.Refusal, match="latest_c_md_2026.py is not the module"): + build.fix_module_pins(real_base.meta, tmp_path) + + +def test_the_fix_entry_tables_and_harness_are_pinned_in_place( + real_base, tmp_path, monkeypatch +): + """Imported in place, latest_final.py, the sales tax tables and sweep.py + are each checked against the bytes committed at the base commit, not + against themselves.""" + import shutil + + for path in build.FIXES.glob("*.py"): + shutil.copy(path, tmp_path / path.name) + shutil.copy(build.SALES_TAX_SOURCE, tmp_path / build.SALES_TAX_TABLES) + monkeypatch.setattr(build, "FIXES", tmp_path) + build.fix_module_pins(real_base.meta, tmp_path) + entry = tmp_path / "latest_final.py" + original = entry.read_bytes() + entry.write_bytes(original + b"# edited\n") + with pytest.raises(build.Refusal, match="latest_final.py is not"): + build.fix_module_pins(real_base.meta, tmp_path) + entry.write_bytes(original) + tables = tmp_path / build.SALES_TAX_TABLES + tables.write_text(tables.read_text() + " ") + monkeypatch.setattr(build, "SALES_TAX_SOURCE", tables) + with pytest.raises(build.Refusal, match="r19_irs_sales_tax_2025.json is not"): + build.fix_module_pins(real_base.meta, tmp_path) + + assert build.harness_pin()["sha256"] == build.sha256(build.HARNESS) + harness = tmp_path / "sweep.py" + harness.write_text(build.HARNESS.read_text() + "# edited\n") + with pytest.raises(build.Refusal, match="sweep.py is not"): + build.harness_pin(harness) + + +def test_the_real_base_on_its_own_values_plans_no_change(real_base): + """A stub engine that returns 20261010's values: the plan accepts the real + release record, which rules nothing, and changes nothing.""" + release, ruled, _ = build.release_record(real_base.exclusions) + computed = real_base.reference.values() + actions = _actions( + kept_exclusions_from_release=[ + {"scenario_id": k[0], "variable": k[1]} for k in sorted(ruled) + ] + ) + plan = build.plan_upgrade(real_base, release, ruled, actions, computed) + assert plan.problems == [] and plan.changed == [] and plan.within == [] + + +# --- Narratives ---------------------------------------------------------------- + + +EXPLANATIONS = ( + "country,scenario_id,variable,reference_value,trace_lines,explanation,error\n" + f'us,scenario_001,{S},1000.0,3,"Old, text.",\n' + "us,scenario_001,local_income_tax,0.0,1,No local tax.,\n" + f'us,scenario_002,{S},500.25,2,"Said ""500.25"".",\n' + "us,scenario_002,federal_income_tax_before_refundable_credits,250.5,4,Fed.,\n" + 'us,scenario_003,snap,3000.0,9,"Multi\nline",\n' + "us,scenario_004,state_income_tax_before_refundable_credits,700.0,1,AZ.,\n" + "us,scenario_006,local_income_tax,0.0,1,No county.,\n" +) + + +class _Response: + def __init__(self, text): + self.choices = [ + types.SimpleNamespace(message=types.SimpleNamespace(content=text)) + ] + + +def _scenarios(): + import pandas as pd + + return pd.DataFrame( + [ + { + "scenario_id": f"scenario_00{i}", + "state": "IN", + "filing_status": "single", + "num_adults": 1, + "num_children": 0, + "total_income": 1000.0, + } + for i in range(1, 7) + ] + ) + + +def _narrate(tmp_path, completion, hand=None): + computed, actions = _full() + del actions["new_exclusions"][0] + computed[("scenario_006", "local_income_tax")] = 0.0 + _write(tmp_path, computed, actions) + return narratives.write_narratives( + tmp_path, EXPLANATIONS, _scenarios(), completion=completion, hand_corrected=hand + ) + + +def test_narratives_rewrite_only_the_changed_rows(tmp_path): + prompts = [] + + def completion(**kwargs): + prompts.append(kwargs) + value = float( + kwargs["messages"][0]["content"] + .split("REFERENCE VALUE: ")[1] + .split("\n")[0] + ) + return _Response(f"# Heading\nPolicyEngine gives ${narratives.money(value)}.") + + text, results = _narrate(tmp_path, completion) + assert {r[:2] for r in results} == { + ("scenario_001", S), + ("scenario_002", F), + ("scenario_004", S), + ("scenario_002", S), + } + assert all(p["model"] == "claude-haiku-4-5" for p in prompts) + assert all("policyengine-us 9.9.9" in p["messages"][0]["content"] for p in prompts) + old = narratives.parse_rows(EXPLANATIONS)[1] + new = narratives.parse_rows(text)[1] + changed = {r[:2] for r in results} + for (fields, raw), (new_fields, new_raw) in zip(old, new, strict=True): + if (fields[1], fields[2]) in changed: + assert new_fields[5].startswith("PolicyEngine gives $") + assert new_fields[4] == "2" and new_fields[6] == "" + else: + assert raw == new_raw + assert f"us,scenario_001,{S},1010.0,2,PolicyEngine gives $1,010." not in text + assert '1010.0,2,"PolicyEngine gives $1,010."' in text + + +def _value_narrator(mention: str = ""): + def completion(**kwargs): + value = float( + kwargs["messages"][0]["content"] + .split("REFERENCE VALUE: ")[1] + .split("\n")[0] + ) + return _Response(f"PolicyEngine gives ${narratives.money(value)}.{mention}") + + return completion + + +def test_a_rebuild_reuses_the_narratives_whose_writer_inputs_are_unchanged(tmp_path): + """Build A on 9.9.9, build B on 9.9.10 with one approved value moved: B + reuses A's narratives for every changed output whose value, cause, + grounding (the engine's name aside), variable and trace are A's, and only + those; a narrative that names A's engine is written again.""" + computed, actions = _full() + del actions["new_exclusions"][0] + computed[("scenario_006", "local_income_tax")] = 0.0 + _write(tmp_path / "a", computed, actions) + text_a, _ = narratives.write_narratives( + tmp_path / "a", EXPLANATIONS, _scenarios(), completion=_value_narrator() + ) + later = copy.deepcopy(actions) + later["engine"] = "policyengine-us 9.9.10" + later["approved"][0]["value"] = 1020.0 + moved = dict(computed) + moved[("scenario_001", S)] = 1020.0 + _write(tmp_path / "b", moved, later) + reuse = narratives.reusable_narratives(tmp_path / "b", tmp_path / "a", text_a) + assert set(reuse) == {("scenario_002", F), ("scenario_004", S), ("scenario_002", S)} + calls = [] + + def counting(**kwargs): + calls.append(kwargs) + return _value_narrator()(**kwargs) + + text_b, results = narratives.write_narratives( + tmp_path / "b", EXPLANATIONS, _scenarios(), completion=counting, reuse=reuse + ) + assert len(calls) == 1 and "1020.0" in calls[0]["messages"][0]["content"] + rows_a = {(f[1], f[2]): f for f, _ in narratives.parse_rows(text_a)[1]} + rows_b = {(f[1], f[2]): f for f, _ in narratives.parse_rows(text_b)[1]} + for key in reuse: + assert rows_b[key] == rows_a[key] + # A narrative that names the earlier engine is not reused. + named, _ = narratives.write_narratives( + tmp_path / "a", + EXPLANATIONS, + _scenarios(), + completion=_value_narrator(" On policyengine-us 9.9.9."), + ) + assert narratives.reusable_narratives(tmp_path / "b", tmp_path / "a", named) == {} + # On the same engine it is still true, so a rebuild there keeps it. + same = narratives.reusable_narratives(tmp_path / "a", tmp_path / "a", named) + assert set(same) == { + ("scenario_001", S), + ("scenario_002", F), + ("scenario_004", S), + ("scenario_002", S), + } + + +@pytest.mark.parametrize( + "text, value, ok", + [ + ("Refundable credits are $0.00.", 0.0, True), + ("Refundable credits are $0.", 0.0, True), + ("In tax year 2026 the household gets nothing.", 0.0, False), + ("Tax is $1,916.61 after credits.", 1916.606201, True), + ("Tax is $1,916 after credits.", 1916.606201, False), + ("The refund is -$12.50.", -12.5, True), + ("It owes $2,116.61 before a $200 credit.", 1916.61, False), + ], +) +def test_a_narrative_states_its_amount_as_dollars_to_the_cent(text, value, ok): + assert narratives.states_amount(text, value) is ok + + +@SETTINGS +@given(cents=st.integers(min_value=0, max_value=10**9)) +def test_a_formatted_amount_is_always_recognized(cents): + value = cents / 100 + assert narratives.states_amount(f"PolicyEngine gives ${value:,.2f}.", value) + assert narratives.states_amount( + f"PolicyEngine gives ${narratives.money(value)}.", value + ) + + +def test_reuse_refuses_a_narratives_file_from_another_build(tmp_path): + """The base explanations are not build A's narratives: their changed rows + carry the old values, so reusing from them is refused.""" + computed, actions = _full() + del actions["new_exclusions"][0] + computed[("scenario_006", "local_income_tax")] = 0.0 + _write(tmp_path / "a", computed, actions) + text_a, _ = narratives.write_narratives( + tmp_path / "a", EXPLANATIONS, _scenarios(), completion=_value_narrator() + ) + assert narratives.reusable_narratives(tmp_path / "a", tmp_path / "a", text_a) + with pytest.raises(build.Refusal, match="is not --reuse-from's narratives"): + narratives.reusable_narratives(tmp_path / "a", tmp_path / "a", EXPLANATIONS) + + +@pytest.mark.parametrize("stated", ["nan", "inf", "-inf", "", "n/a"]) +def test_reuse_refuses_a_source_value_that_is_not_a_finite_number(tmp_path, stated): + """The delta review's case: abs(nan - x) > 1e-6 is false, so a source + narrative row whose reference_value reads "nan" would pass a bare + comparison and its (possibly stale) text be reused.""" + computed, actions = _full() + del actions["new_exclusions"][0] + computed[("scenario_006", "local_income_tax")] = 0.0 + _write(tmp_path / "a", computed, actions) + text_a, _ = narratives.write_narratives( + tmp_path / "a", EXPLANATIONS, _scenarios(), completion=_value_narrator() + ) + rows = {(f[1], f[2]): (f, raw) for f, raw in narratives.parse_rows(text_a)[1]} + fields, raw = rows[("scenario_002", F)] + edited = raw.replace(f",{fields[3]},", f",{stated},", 1) + assert edited != raw + with pytest.raises(build.Refusal, match="is not --reuse-from's narratives"): + narratives.reusable_narratives( + tmp_path / "a", tmp_path / "a", text_a.replace(raw, edited) + ) + + +def test_a_narrative_that_omits_the_value_is_retried_then_refused(tmp_path): + calls = [] + + def stubborn(**kwargs): + calls.append(kwargs) + return _Response("No figure here.") + + with pytest.raises(build.Refusal, match="omits"): + _narrate(tmp_path, stubborn) + assert len(calls) == 1 + narratives.RETRIES + assert "must state the value" in calls[-1]["messages"][0]["content"] + + +def test_hand_corrected_narratives_must_state_the_value_and_be_changed(tmp_path): + def completion(**kwargs): + value = float( + kwargs["messages"][0]["content"] + .split("REFERENCE VALUE: ")[1] + .split("\n")[0] + ) + return _Response(f"Gives ${narratives.money(value)}.") + + hand = {f"scenario_001|{S}": "Corrected: $1,010 of tax."} + text, _ = _narrate(tmp_path / "a", completion, hand) + assert "Corrected: $1,010 of tax." in text + with pytest.raises(build.Refusal, match="did not change"): + _narrate(tmp_path / "b", completion, {"scenario_003|snap": "x"}) + with pytest.raises(build.Refusal, match="omits"): + _narrate(tmp_path / "c", completion, {f"scenario_001|{S}": "No value."}) + + +def test_a_new_exclusion_is_grounded_on_its_alternative_reading(): + item = { + "scenario_id": "s", + "variable": "v", + "cause": "excluded_reference_depends_on_unlisted_input", + "basis": "b", + } + record = { + "reason_code": "reference_depends_on_unlisted_input", + "alternative_reading": "Read so.", + } + text = narratives.grounding_for(item, {("s", "v"): record}) + assert text.endswith("an input the prompt does not state. Read so.") + assert narratives.grounding_for({**item, "cause": "a"}, {("s", "v"): record}) == "b" + + +def test_the_writer_is_anthropics(tmp_path, monkeypatch): + import policybench.case_reference_explanations as writer + + monkeypatch.setattr(writer, "REFERENCE_MODEL", "gpt-6-luna") + with pytest.raises(build.Refusal, match="Anthropic"): + _narrate(tmp_path, lambda **_: _Response("")) + + +def test_the_narrative_run_never_uses_an_openai_key(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + import os + + with pytest.raises(build.Refusal, match="ANTHROPIC_API_KEY"): + narratives.main(["--references", "x", "--explanations", "y", "--out", "z"]) + assert "OPENAI_API_KEY" not in os.environ + + +def test_the_committed_explanations_round_trip_byte_for_byte(): + path = ROOT / "annotations" / "us_full_run_20260612_policyengine_4_16_1_populace" + text = (path / "us_case_reference_explanations.csv").read_text() + assert narratives.rewrite_explanations(text, {}) == text + + +# --- Draft actions --------------------------------------------------------------- + + +def _cells(computed, states=None): + base = _base() + states = states or {} + return [ + drafting.Cell( + r.key[0], + states.get(r.key, "XX"), + r.key[1], + r.value, + computed[r.key], + "", + "", + ) + for r in base.reference.rows + ] + + +def test_the_draft_places_every_moved_output_and_prints_each_call(tmp_path): + computed, _ = _full() + computed[("scenario_001", "local_income_tax")] = 670.55 # IN county + computed[("scenario_006", "local_income_tax")] = 0.0 + states = { + ("scenario_001", "local_income_tax"): "IN", + ("scenario_001", S): "ZZ", + } + base = _base() + release, ruled = _release(base) + actions, calls = drafting.draft_actions( + _cells(computed, states), + release, + ruled, + {build.key_of(r) for r in base.exclusions["exclusions"]}, + {}, + engine_version="9.9.9", + date=DATE, + spec_sha256="spec", + precise=True, + ) + assert actions["draft"] is True + placed = { + (c["scenario_id"], c["variable"]): (c["decision"], c["placed_in"]) + for c in calls + } + assert placed == { + ("scenario_001", S): ("UNDECIDED", "review only"), + ("scenario_001", "local_income_tax"): ("PROPOSED", "new_exclusions"), + ("scenario_002", F): ("PROPOSED", "regenerated_exclusions"), + ("scenario_004", S): ("PROPOSED", "regenerated_exclusions"), + ("scenario_003", "snap"): ("PROPOSED", "excluded_rechecked"), + } + assert actions["kept_exclusions_from_release"] == [ + {"scenario_id": "scenario_004", "variable": F}, + {"scenario_id": "scenario_005", "variable": S}, + ] + # The records name no upstream fix, so the builder refuses the regenerations + # until a reviewer names them; cleared, the draft builds except for the + # undecided move. + cleared = {k: v for k, v in actions.items() if k not in ("draft", "review")} + build.validate_actions(cleared) + problems = build.plan_upgrade(base, release, ruled, cleared, computed).problems + assert problems == [ + f"unreviewed move ('scenario_001', '{S}'): 1000.0 -> 1010.0", + f"regenerated ('scenario_002', '{F}') names no upstream fix ('to be filed')", + f"regenerated ('scenario_004', '{S}') names no upstream fix ('to be filed')", + ] + for entry in cleared["regenerated_exclusions"]: + entry["upstream"] = "PolicyEngine/policyengine-us#1" + problems = build.plan_upgrade(base, release, ruled, cleared, computed).problems + assert problems == [f"unreviewed move ('scenario_001', '{S}'): 1000.0 -> 1010.0"] + + +def test_the_draft_keeps_an_on_target_record_that_names_a_second_reason(): + """A defect record whose note also names an unlisted input stays excluded + (drafted as a recheck), even when the engine lands on its corrected value.""" + computed = _unchanged() + computed[("scenario_002", F)] = 300.0 + base = _base() + release, ruled = _release(base) + for record in release["exclusions"]: + if build.key_of(record) == ("scenario_002", F): + record["note"] = "The output also moves under r23 (unlisted input)." + actions, calls = drafting.draft_actions( + _cells(computed), + release, + ruled, + {build.key_of(r) for r in base.exclusions["exclusions"]}, + {}, + engine_version="9.9.9", + date=DATE, + spec_sha256="spec", + precise=True, + ) + assert actions["regenerated_exclusions"] == [] + (entry,) = actions["excluded_rechecked"] + assert "names a second reason" in entry["reason"] + assert calls[0]["placed_in"] == "excluded_rechecked" + + +def test_the_draft_aims_a_drifted_record_at_its_fix_modules_evidence(): + """scenario_003 snap misses its record's 1.755.4 corrected value; with + evidence it is drafted against the modules' pre-fix corrected value, and + the builder accepts the draft once the modules leave it unchanged.""" + computed = _unchanged() + computed[("scenario_003", "snap")] = 2600.0 + base = _base() + release, ruled = _release(base) + kwargs = dict(engine_version="9.9.9", date=DATE, spec_sha256="spec", precise=True) + args = ( + _cells(computed), + release, + ruled, + {build.key_of(r) for r in base.exclusions["exclusions"]}, + {}, + ) + without, _ = drafting.draft_actions(*args, **kwargs) + assert without["regenerated_exclusions"] == [] + assert "$100.00 from the record's" in without["excluded_rechecked"][0]["reason"] + doc = _evidence() + actions, calls = drafting.draft_actions( + *args, **kwargs, evidence={"ref": dict(EVIDENCE), "doc": doc} + ) + (entry,) = actions["regenerated_exclusions"] + assert entry["target"] == { + "kind": "fix_modules", + "modules": [MODULE], + "evidence": EVIDENCE, + } + assert entry["alternative_value"] == 2500.0 + assert "$2,600.30 its fix modules (r_fix_v2.py) give on" in entry["basis"] + assert [c["placed_in"] for c in calls] == ["regenerated_exclusions"] + cleared = {k: v for k, v in actions.items() if k not in ("draft", "review")} + entry["upstream"] = "PolicyEngine/policyengine-us#1" + plan = build.plan_upgrade( + base, + release, + ruled, + cleared, + computed, + evidence={EVIDENCE["path"]: doc}, + module_pins={MODULE: "p" * 64}, + module_values={(("scenario_003", "snap"), (MODULE,)): 2600.0}, + ) + assert plan.problems == [] + assert [r["target"]["kind"] for r in plan.regenerated] == ["fix_modules"] + + +def test_a_regeneration_must_name_its_upstream_fix(): + computed = _unchanged() + computed[("scenario_002", F)] = 300.0 + for upstream in ("", "to be filed", "To be filed "): + entry = {**_regenerate("scenario_002", F, 300.0), "upstream": upstream} + (problem,) = _problems(computed, _actions(regenerated_exclusions=[entry])) + assert "names no upstream fix" in problem + + +def test_the_draft_rules_recognize_the_documented_scored_moves(): + county = drafting.Cell( + "scenario_015", "IN", "local_income_tax", 0.0, 670.55, "scored", "" + ) + fund = drafting.Cell("scenario_076", "ID", S, 6818.34, 6828.34, "scored", "") + other = drafting.Cell("scenario_076", "ID", S, 6818.34, 6838.34, "scored", "") + proposals = [ + [ + propose(c, ENGINE, DATE)["rule"] + for matches, propose in drafting.RULES + if matches(c) + ] + for c in (county, fund, other) + ] + assert proposals == [ + ["indiana_county_unlisted"], + ["idaho_permanent_building_fund"], + [], + ] + record = drafting._indiana_county(county, ENGINE, DATE)["entry"] + from policybench.reference_exclusions import REQUIRED_FIELDS_BY_REASON + + assert all( + record[f] not in (None, "") + for f in REQUIRED_FIELDS_BY_REASON[record["reason_code"]] + ) + + +def test_the_cells_table_must_be_against_20261010(tmp_path): + board = {("scenario_001", S): 1000.0} + path = tmp_path / "cells_pe2.37.1.csv" + path.write_text( + "scenario_id,state,variable,frozen_20261010,new,delta,moved,status,audited_target\n" + f"scenario_001,IN,{S},999.0,1000.0,1.0,False,scored,\n" + ) + with pytest.raises(build.Refusal, match="not 20261010's"): + drafting.read_cells(path, board, None) + assert drafting.engine_from_path(path) == "2.37.1" + path.write_text(path.read_text().replace("999.0", "1000.0")) + (cell,) = drafting.read_cells(path, board, None) + assert cell.new == 1000.0 and cell.state == "IN" + with pytest.raises(build.Refusal, match="not the table's"): + drafting.read_cells(path, board, {("scenario_001", S): 1000.5}) + + +def test_a_regeneration_names_the_upstream_fix_the_map_gives(): + upstreams = { + "outputs": {"scenario_064|state": "PolicyEngine/policyengine-us#9801"}, + "root_causes": {"r02": "#10032", "r11": "#10034"}, + } + record = {"root_cause": "r02+r11", "upstream": "to be filed"} + assert drafting.upstream_for(record, ("scenario_005", "state"), upstreams) == ( + "#10032; #10034" + ) + assert drafting.upstream_for(record, ("scenario_064", "state"), upstreams) == ( + "PolicyEngine/policyengine-us#9801" + ) + # A root cause the map lacks keeps the record's own upstream. + partial = {"root_cause": "r02+r99", "upstream": "PolicyEngine/policyengine-us#1"} + assert drafting.upstream_for(partial, ("s", "v"), upstreams) == ( + "PolicyEngine/policyengine-us#1" + ) + assert drafting.upstream_for(record, ("s", "v"), None) == "to be filed" + + +def test_the_committed_upstream_map_names_every_pending_root_cause(): + """Every 20261010 engine-defect root cause maps to a fix (MA 081's r22 + stays excluded for its second reason, so it needs none), and CA 099's + state income tax names #10052 as well.""" + upstreams = json.loads((UPGRADE / "upstreams.json").read_text()) + base = build.load_base() + causes = { + part + for record in base.exclusions["exclusions"] + if record["reason_code"] == build.ENGINE_DEFECT + for part in record["root_cause"].split("+") + } + assert causes - set(upstreams["root_causes"]) == {"r22_ma_part_a_loss_offset"} + + +# --- This audit's own fix modules (r33) ---------------------------------------- + + +R33 = "r33_ca_educator_expense_addback.py" + + +def test_a_module_absent_at_the_base_is_read_from_this_audits_fixes(): + """r33 is not in reference_audit/2026-09-22/fixes at BASE_COMMIT, so it is + this audit's: its path is LOCAL_FIXES_REL and its bytes are HEAD's.""" + rel, blob = build.fix_location(R33) + assert rel == f"{build.LOCAL_FIXES_REL}/{R33}" + assert blob == (build.LOCAL_FIXES / R33).read_bytes() + assert build.is_local_fix(R33) + rel, _ = build.fix_location("r02_ira_219g_v2.py") + assert rel == f"{build.AUDIT_FIXES_REL}/r02_ira_219g_v2.py" + assert not build.is_local_fix("r02_ira_219g_v2.py") + assert not build.is_local_fix("no_such_module.py") + with pytest.raises(build.Refusal, match="neither in"): + build.fix_location("no_such_module.py") + + +def test_a_local_modules_pin_names_its_path(): + pins = build.audit_module_pins(["r02_ira_219g_v2.py", R33]) + assert pins[R33] == build.sha256(build.LOCAL_FIXES / R33) + assert build.module_pin(R33, pins) == { + "module": R33, + "sha256": pins[R33], + "path": f"{build.LOCAL_FIXES_REL}/{R33}", + } + assert set(build.module_pin("r02_ira_219g_v2.py", pins)) == {"module", "sha256"} + + +def test_a_local_module_edited_after_its_commit_is_refused(tmp_path, monkeypatch): + copy_dir = tmp_path / "fixes" + copy_dir.mkdir() + (copy_dir / R33).write_bytes((build.LOCAL_FIXES / R33).read_bytes() + b"# edit\n") + monkeypatch.setattr(build, "LOCAL_FIXES", copy_dir) + with pytest.raises(build.Refusal, match="as committed at HEAD"): + build.audit_module_pins([R33]) + + +def test_the_committed_evidence_targets_ca_099_with_r33(): + """evidence/pe2.38.6.json gives CA 099's state income tax with r02, r11 and + r33 on an engine with the defect: the addback moves it $27.00, from + $4,588.48 to $4,615.48 (8% of the spouse's $337.50 educator expense).""" + doc = json.loads((UPGRADE / "evidence" / "pe2.38.6.json").read_text()) + assert doc["engine"] == "policyengine-us 2.38.6" + (item,) = [ + i + for i in doc["items"] + if (i["scenario_id"], i["variable"]) == ("scenario_099", S) + ] + assert item["modules"] == ["r02_ira_219g_v2.py", "r11_ca_itemized_conformity_v2.py", R33] + assert round(item["corrected_value"] - item["engine_value"], 2) == 27.00 + assert doc["modules"][R33] == build.sha256(build.LOCAL_FIXES / R33) + + +def test_the_freeze_accepts_a_target_module_only_under_an_audits_fixes(): + sys.path.insert(0, str(ROOT / "scripts")) + import freeze_snapshot + + target = { + "kind": "fix_modules", + "modules": [ + { + "module": R33, + "sha256": build.sha256(build.LOCAL_FIXES / R33), + "path": f"{build.LOCAL_FIXES_REL}/{R33}", + } + ], + "dependencies": [], + "evidence": "reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json", + "evidence_sha256": build.sha256(UPGRADE / "evidence" / "pe2.38.6.json"), + } + upgrade = { + "regenerated_exclusions": [ + {"scenario_id": "scenario_099", "variable": S, "target": target} + ] + } + freeze_snapshot.verify_regeneration_target_pins(upgrade) + target["modules"][0]["path"] = f"scripts/{R33}" + with pytest.raises(SystemExit, match="not an audit's fixes/ directory"): + freeze_snapshot.verify_regeneration_target_pins(upgrade) diff --git a/tests/test_finish_musewonder.py b/tests/test_finish_musewonder.py new file mode 100644 index 00000000..4867fb50 --- /dev/null +++ b/tests/test_finish_musewonder.py @@ -0,0 +1,6635 @@ +"""The Muse Wonder addition moves an incumbent only as the ten records say. + +Adapted from test_finish_gpt61sol.py. Synthetic data stands in for runs, +audits and payloads. Release 20261010 is read from git at BASE_COMMIT, never +from the working tree (the freeze rewrites it). Tests that read the seed +copy, the main clone's grounding or the live stage read them only, and skip +where they are absent. +""" + +from __future__ import annotations + +import contextlib +import copy +import csv +import functools +import gzip +import hashlib +import inspect +import io +import json +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pandas as pd +import pytest +from hypothesis import HealthCheck, assume, given, settings +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import finish_gpt61sol as gpt61sol # noqa: E402 +import finish_musewonder as driver # noqa: E402 +import release_20261006 # noqa: E402 + +from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 + +NEW = "muse-wonder" +SLUG = "musewonder" +REPO = Path(__file__).resolve().parents[1] +BASE_COMMIT = driver.BASE_COMMIT +COMMITTED_SEED_DIGEST_SHA256 = driver.SEED_DIGEST_SHA256 +SNAPSHOT_PATH = Path("paper/snapshot/20260501/runs") / driver.RUN_NAME +ADJUDICATIONS_PATH = Path("annotations") / driver.RUN_NAME / driver.ADJUDICATIONS +EXCLUSIONS_NAME = "reference_exclusions.json" +BOUND_EXCLUSIONS = f"publish/{driver.RUN_NAME}/us/{EXCLUSIONS_NAME}" +# Release 20261010's audit (the seed, a read-only copy), the grounding it was +# rendered with, the supervised runs root and the live stage. All local only; +# nothing here writes to them. +SEED = REPO / "results/local/musewonder/seed-20261010/audit" +STAGE = REPO / "results/local/musewonder/stage" +# The working stage carrying the engine upgrade of the references: an APFS +# clone of STAGE with a reference build installed (local only). STAGE stays +# the 2.15.17 fallback. The committed judge provenance record describes it. +REHEARSAL = REPO / "results/local/rehearsal-2372" +WORKING_STAGE = REHEARSAL / "stage" +GROUNDING = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/" + "grounding.csv" +) +RUNS = Path("/Users/maxghenis/PolicyEngine/policybench/results/local/musewonder-runs") +# The UTC day a test's adjudicate step says its decisions were written on. +WRITTEN_ON = "2026-10-09" +# The seven fields the ruling decides on an entry. +DECISION_FIELDS = ( + "adjudicated_failure_source", + "adjudicated_failure_subtype", + "adjudicated_on", + "adjudicator", + "excluded_from_scoring", + "reference_verdict", + "reference_basis", +) + + +@functools.cache +def base_blob(path: Path) -> bytes: + """A repository file as committed at BASE_COMMIT (release 20261010).""" + result = subprocess.run( + ["git", "-C", str(REPO), "show", f"{BASE_COMMIT}:{path.as_posix()}"], + capture_output=True, + ) + if result.returncode: + pytest.fail( + f"cannot read {path} at base commit {BASE_COMMIT[:12]}; fetch full " + f"history (git fetch --unshallow): {result.stderr.decode().strip()}" + ) + return result.stdout + + +def base_references(directory: Path) -> Path: + """Write release 20261010's five reference files into ``directory``.""" + directory.mkdir(parents=True, exist_ok=True) + for name in driver.REFERENCE_FILES: + (directory / name).write_bytes(base_blob(SNAPSHOT_PATH / name)) + return directory + + +def base_record() -> dict: + """Release 20261010's exclusion record, a fresh copy each call.""" + return json.loads(base_blob(SNAPSHOT_PATH / EXCLUSIONS_NAME)) + + +def base_adjudication_record() -> dict: + """Release 20261010's adjudication record file, a fresh copy each call.""" + return json.loads(base_blob(ADJUDICATIONS_PATH)) + + +def real_spec() -> dict: + """docs/musewonder/spec.json as the working tree holds it.""" + return json.loads((REPO / driver.SPEC_PATH).read_text()) + + +@functools.cache +def _release_text() -> str: + with mock.patch.object(driver, "ROOT", REPO): + record = driver.build_release_exclusions(base_record(), real_spec()) + return driver.exclusions_text(record) + + +def release_text() -> str: + """The release's exclusion record as install-exclusions writes it, + computed against the real repository whatever a test patched.""" + return _release_text() + + +def release_sha() -> str: + return hashlib.sha256(release_text().encode()).hexdigest() + + +def ruled_case(item: dict) -> str: + return f"us__{item['scenario_id']}__{item['variable']}" + + + + +@pytest.fixture +def unruled_spec(monkeypatch): + """A spec whose rulings decide no adjudication, so the gates ported from + the GPT-6.1 Sol release see their own synthetic records only.""" + spec = { + **real_spec(), + "adjudications_written_on": WRITTEN_ON, + "adjudications": [], + "restated_adjudications": [], + } + monkeypatch.setattr(driver, "load_spec", lambda: copy.deepcopy(spec)) + return spec + + +def test_the_addition_is_muse_wonder_alone_on_a_48_model_board(): + assert driver.MODELS == {SLUG: NEW} + assert driver.BASE_MODELS == 47 and driver.BOARD_MODELS == 48 + assert driver.BASE_OUTPUTS == 1984 + assert driver.BASE_EXCLUSIONS == 58 and driver.BASE_SCORED == 1926 + assert driver.NEW_EXCLUSIONS == 0 and driver.RELEASE_EXCLUSIONS == 58 + assert driver.RELEASE_SCORED == 1926 + assert driver.BASE_TAG == "dashboard-data-20261010" + assert driver.BASE_COMMIT == "5a8164a001efb27fa55f47fe7ea26666a0de31f8" + assert driver.SEED_RELEASE_COMMIT == driver.BASE_COMMIT + assert set(driver.BASE_REFERENCE_SHA256) == set(driver.REFERENCE_FILES) + assert driver.JUDGE_MODEL == "claude-opus-5-5" + assert driver.JUDGE_TEMPLATE_VERSION == 2 + assert driver.REWORDED_SINCE_SEED == frozenset() + assert driver.BASE_ENGINE == "policyengine-us 2.38.6" + + +def test_the_spec_names_this_drivers_release_and_base(): + """The freeze compares the spec's tag and base with the driver's.""" + spec = real_spec() + assert spec["release_tag"] == driver.RELEASE_TAG + assert spec["base_tag"] == driver.BASE_TAG + assert spec["base_sha256"] == driver.BASE_SHA256 + + +def test_the_base_pointer_at_the_base_commit_is_release_20261010(): + """BASE_TAG and BASE_SHA256 are what PR #208's merge commits as the live + dashboard pointer.""" + pointer = json.loads(base_blob(Path("app/src/data.artifact.json"))) + assert pointer["tag"] == driver.BASE_TAG + assert pointer["sha256"] == driver.BASE_SHA256 + + +# --- Run discovery ------------------------------------------------------------- + + +@pytest.fixture +def runs(tmp_path): + root = tmp_path / "runs" + directory = root / SLUG / "run" + directory.mkdir(parents=True) + (directory / "run_state.json").write_text( + json.dumps( + {"model": NEW, "completed": 100, "total": 100, "stopped_reason": None} + ) + ) + (directory / "predictions.csv").write_text("model,scenario_id,variable\n") + return root + + +def change_state(root, **changes): + path = root / SLUG / "run/run_state.json" + state = json.loads(path.read_text()) + state.update(changes) + path.write_text(json.dumps(state)) + + +def test_discovers_only_the_registered_model(runs): + extra = runs / "sonnet55/run" + extra.mkdir(parents=True) + (extra / "run_state.json").write_text(json.dumps({"model": "claude-sonnet-5.5"})) + found = driver.discover_new_models(runs) + assert [(run.slug, run.model) for run in found] == [(SLUG, NEW)] + assert found[0].predictions.parent == found[0].run_dir + + +@pytest.mark.parametrize( + "changes", + [ + {"completed": 99}, + {"completed": 38}, + {"stopped_reason": "budget exhausted"}, + {"stopped_reason": ""}, + {"stopped_reason": False}, + {"completed": 0, "total": 0}, + {"completed": -1, "total": -1}, + {"completed": True, "total": True}, + {"completed": 100.0, "total": 100.0}, + {"completed": "100", "total": "100"}, + ], +) +def test_an_incomplete_or_stopped_run_is_refused(runs, changes): + change_state(runs, **changes) + with pytest.raises(SystemExit, match="refusing incomplete additions"): + driver.discover_new_models(runs) + + +@pytest.mark.parametrize( + "model", ["claude-haiku-5", "claude-haiku-5-5", "claude-sonnet-5.5", "", None] +) +def test_the_wrong_model_in_a_run_state_is_refused(runs, model): + change_state(runs, model=model) + with pytest.raises(SystemExit, match="unexpected model"): + driver.discover_new_models(runs) + + +@pytest.mark.parametrize("name", ["run_state.json", "predictions.csv"]) +def test_a_missing_run_artifact_is_refused(runs, name): + (runs / SLUG / "run" / name).unlink() + with pytest.raises(SystemExit, match=SLUG): + driver.discover_new_models(runs) + + +def test_scratch_completion_still_requires_completed_equal_total(runs): + change_state(runs, completed=4, total=5, synthetic_partial=True) + with pytest.raises(SystemExit, match="4/5"): + driver.discover_new_models(runs) + change_state(runs, completed=5) + assert len(driver.discover_new_models(runs)) == 1 + + +# --- Input pins ---------------------------------------------------------------- + + +def test_pin_inputs_writes_the_finished_runs_file_hashes(runs, tmp_path, monkeypatch): + """--step pin-inputs needs no stage: it hashes the finished run's + predictions.csv and run_state.json and refuses an unfinished run.""" + pins = tmp_path / "input_pins.json" + monkeypatch.setattr(driver, "INPUT_PINS", pins) + driver.main(["--step", "pin-inputs", "--runs-root", str(runs)]) + record = json.loads(pins.read_text()) + assert record == driver.input_pins_record(runs) + run = runs / SLUG / "run" + assert record["inputs"] == { + SLUG: { + "run": f"runs/{SLUG}/run", + "sha256": { + name: hashlib.sha256((run / name).read_bytes()).hexdigest() + for name in ("predictions.csv", "run_state.json") + }, + } + } + change_state(runs, completed=99) + with pytest.raises(SystemExit, match="refusing incomplete additions"): + driver.main(["--step", "pin-inputs", "--runs-root", str(runs)]) + + +def test_the_committed_input_pins_name_both_run_files(): + """Runs anywhere: the committed pins name Muse Wonder's two run + files, as the design note gives them.""" + record = json.loads(driver.INPUT_PINS.read_text()) + assert set(record["inputs"]) == {SLUG} + assert record["inputs"][SLUG]["run"] == f"musewonder-runs/{SLUG}/run" + pins = record["inputs"][SLUG]["sha256"] + assert set(pins) == {"predictions.csv", "run_state.json"} + assert all(re.fullmatch("[0-9a-f]{64}", value) for value in pins.values()) + assert pins["predictions.csv"].startswith("27a5e48f") + assert pins["predictions.csv"].endswith("8344") + assert pins["run_state.json"].startswith("42f82f50") + assert pins["run_state.json"].endswith("bd4e") + assert driver.INPUT_PINS.read_text() == ( + json.dumps(record, indent=2, sort_keys=True) + "\n" + ) + + +def test_the_committed_input_pins_are_the_finished_runs(): + """Local only: recomputed from the run directory, the pins are the same.""" + if not (RUNS / SLUG / "run").is_dir(): + pytest.skip("Muse Wonder's run directory is not on this machine") + assert driver.input_pins_record(RUNS) == json.loads(driver.INPUT_PINS.read_text()) + + +@pytest.mark.slow +def test_the_stages_inputs_are_the_pinned_run(): + """Local only, read-only: the stage's copies are the pinned bytes, its + prepare-time hashes hold, and its bundle rows are the run file's in every + column (this reads the stage's 700 MB bundle predictions).""" + if not (STAGE / "inputs" / SLUG).is_dir(): + pytest.skip("needs the Muse Wonder stage") + pins = json.loads(driver.INPUT_PINS.read_text())["inputs"][SLUG]["sha256"] + for name, pin in pins.items(): + assert driver.digest(STAGE / "inputs" / SLUG / name) == pin + driver.verify_prepared_inputs(STAGE) + assert driver.new_model_row_differences(STAGE) == [] + + +def commit(root: Path, message: str) -> str: + """Commit everything in a scratch repository, with no user hooks or keys; + return the new commit's sha.""" + git = ["git", "-C", str(root), "-c", "core.hooksPath=/dev/null"] + git += ["-c", "commit.gpgsign=false", "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "--allow-empty", "-qm", message], check=True) + return subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + check=True, + text=True, + ).stdout.strip() + + +@pytest.fixture +def pinned_repo(tmp_path, monkeypatch): + """A git repository whose HEAD commits an input-pins file.""" + root = tmp_path / "repo" + pins = root / driver.INPUT_PINS_PATH + pins.parent.mkdir(parents=True) + record = {"inputs": {SLUG: {"sha256": {"predictions.csv": "a" * 64}}}} + record["inputs"][SLUG]["sha256"]["run_state.json"] = "b" * 64 + pins.write_text(json.dumps(record)) + subprocess.run(["git", "-C", str(root), "init", "-q"], check=True) + commit(root, "pins") + monkeypatch.setattr(driver, "ROOT", root) + monkeypatch.setattr(driver, "INPUT_PINS", pins) + return pins, record + + +def test_the_input_pins_are_read_as_committed_at_head(pinned_repo): + pins, record = pinned_repo + assert driver.committed_input_pins() == { + SLUG: {"predictions.csv": "a" * 64, "run_state.json": "b" * 64} + } + # Pins rewritten and not committed are refused, not silently ignored. + record["inputs"][SLUG]["sha256"]["predictions.csv"] = "c" * 64 + pins.write_text(json.dumps(record)) + with pytest.raises(SystemExit, match="differs from its HEAD commit; commit it"): + driver.committed_input_pins() + + +def test_input_pins_never_committed_are_refused(pinned_repo, monkeypatch): + pins, _ = pinned_repo + elsewhere = pins.with_name("other_pins.json") + elsewhere.write_text(pins.read_text()) + monkeypatch.setattr(driver, "INPUT_PINS_PATH", "docs/musewonder/other_pins.json") + monkeypatch.setattr(driver, "INPUT_PINS", elsewhere) + with pytest.raises(SystemExit, match="is not committed at HEAD"): + driver.committed_input_pins() + + +@pytest.mark.parametrize( + "inputs", + [ + {}, + {SLUG: {"sha256": {"predictions.csv": "a" * 64}}}, + {SLUG: {"sha256": {"predictions.csv": "a", "run_state.json": "b"}}, "x": {}}, + ], +) +def test_input_pins_that_miss_a_run_file_are_refused(pinned_repo, inputs): + pins, _ = pinned_repo + pins.write_text(json.dumps({"inputs": inputs})) + commit(driver.ROOT, "other pins") + with pytest.raises(SystemExit, match="does not pin"): + driver.committed_input_pins() + + +# --- Stage isolation ----------------------------------------------------------- + + +@pytest.fixture +def workspace(tmp_path, monkeypatch): + root = tmp_path / "workspace" + root.mkdir() + monkeypatch.setattr(driver, "ROOT", root) + return root + + +@pytest.mark.parametrize("relation", ["same", "parent", "child"]) +def test_stage_must_not_overlap_any_input(workspace, relation): + stage = workspace / "results/local/stage" + source = {"same": stage, "parent": stage.parent, "child": stage / "source"}[ + relation + ] + with pytest.raises(SystemExit, match="overlaps input"): + driver.validate_stage_path(stage, [source]) + + +@pytest.mark.parametrize("relative", ["app/src", "paper/snapshot", "results/local"]) +def test_stage_must_be_a_scratch_subdirectory(workspace, relative): + with pytest.raises(SystemExit, match="stage-dir must be below"): + driver.validate_stage_path(workspace / relative, []) + + +def test_stage_cannot_contain_symlink_to_a_live_file(workspace): + stage = workspace / "results/local/stage" + stage.mkdir(parents=True) + (stage / "predictions.csv").symlink_to(workspace / "live.csv") + with pytest.raises(SystemExit, match="symlink in stage-dir"): + driver.validate_stage_path(stage, []) + + +def test_disjoint_scratch_stage_is_allowed(workspace): + driver.validate_stage_path( + workspace / "results/local/stage", [workspace / "results/local/inputs"] + ) + + +# --- Prediction keys ----------------------------------------------------------- + + +@pytest.fixture +def predictions(): + return pd.DataFrame( + { + "model": [NEW, NEW], + "scenario_id": ["scenario_000", "scenario_000"], + "variable": ["snap", "ssi"], + "prediction": [0, 1], + } + ) + + +def test_equal_row_counts_do_not_hide_substituted_output_keys(predictions): + wrong = predictions.copy() + wrong.loc[1, "variable"] = "medicaid" + with pytest.raises(SystemExit, match="keys differ"): + driver.validate_keys(wrong, predictions, NEW) + + +def test_duplicate_output_keys_are_rejected(predictions): + repeated = pd.concat([predictions, predictions.iloc[:1]], ignore_index=True) + with pytest.raises(SystemExit, match="duplicate prediction keys"): + driver.validate_keys(repeated, predictions, NEW) + + +def test_predictions_cannot_be_labeled_as_another_model(predictions): + with pytest.raises(SystemExit, match="unexpected model"): + driver.validate_keys(predictions, predictions, "claude-sonnet-5.5") + + +# --- CLI ----------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "options, message", + [ + (["--partial"], "--partial requires --early"), + (["--early", "--step", "judge"], "--early only applies to prepare"), + (["--early", "--step", "triage"], "--early only applies to prepare"), + (["--early", "--step", "export"], "--early only applies to prepare"), + ( + ["--early", "--step", "install-exclusions"], + "--early only applies to prepare", + ), + ( + ["--early", "--step", "adjudicate-exclusions"], + "--early only applies to prepare", + ), + ([], "prepare requires --runs-root"), + ], +) +def test_cli_rejects_unsafe_mode_combinations(tmp_path, options, message, capsys): + with pytest.raises(SystemExit): + driver.parse_args(["--stage-dir", str(tmp_path), *options]) + assert message in capsys.readouterr().err + + +def test_partial_mode_is_explicitly_early(tmp_path): + args = driver.parse_args( + [ + "--stage-dir", + str(tmp_path / "stage"), + "--runs-root", + str(tmp_path / "runs"), + "--early", + "--partial", + ] + ) + assert args.partial and args.early and args.step == "prepare" + + +@pytest.mark.parametrize("step", ["install-exclusions", "adjudicate-exclusions"]) +def test_the_new_steps_need_a_stage(step, capsys): + with pytest.raises(SystemExit): + driver.parse_args(["--step", step]) + assert f"{step} requires --stage-dir" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "step", ["export", "triage", "install-exclusions", "adjudicate-exclusions"] +) +@pytest.mark.parametrize("field", ["partial", "early"]) +def test_an_early_or_partial_stage_cannot_resume_into_release(workspace, field, step): + stage = workspace / "results/local/stage" + stage.mkdir(parents=True) + receipt = {"partial": False, "early": False, "files": {}} + receipt[field] = True + (stage / "stage.json").write_text(json.dumps(receipt)) + with pytest.raises(SystemExit, match="cannot become a release"): + driver.main(["--stage-dir", str(stage), "--step", step]) + + +def test_resume_refuses_changed_staged_inputs(workspace): + stage = workspace / "results/local/stage" + stage.mkdir(parents=True) + (stage / "predictions.csv").write_text("changed data") + (stage / "stage.json").write_text( + json.dumps( + { + "partial": False, + "early": False, + "files": { + "predictions.csv": hashlib.sha256(b"original data").hexdigest() + }, + } + ) + ) + with pytest.raises(SystemExit, match="staged input changed"): + driver.main(["--stage-dir", str(stage), "--step", "export"]) + + +def _resumable(stage: Path, installed: dict | None) -> None: + stage.mkdir(parents=True) + receipt = {"partial": False, "early": False, "files": {}} + if installed is not None: + receipt["exclusions_installed"] = installed + (stage / "stage.json").write_text(json.dumps(receipt)) + + +@pytest.mark.parametrize("step", ["triage", "export", "adjudicate-exclusions"]) +@pytest.mark.parametrize("installed", [None, {}, {"sha256": "0" * 64}]) +def test_later_steps_refuse_a_stage_without_the_installed_exclusions( + workspace, monkeypatch, step, installed +): + sha = release_sha() + monkeypatch.setattr(driver, "release_exclusions_sha256", lambda: sha) + for name in ("triage", "export", "adjudicate_exclusions", "resolve_live_base"): + monkeypatch.setattr( + driver, name, lambda *a, n=name: pytest.fail(f"{n} ran"), raising=True + ) + stage = workspace / "results/local/stage" + _resumable(stage, installed) + with pytest.raises(SystemExit, match="run --step install-exclusions"): + driver.main(["--stage-dir", str(stage), "--step", step]) + + +@pytest.mark.parametrize( + "step, called", + [ + ("triage", "triage"), + ("adjudicate-exclusions", "adjudicate_exclusions"), + ("judge", "judge"), + ("install-exclusions", "install_exclusions"), + ], +) +def test_a_stage_with_the_release_exclusions_reaches_its_step( + workspace, monkeypatch, step, called +): + """The installed record lets triage and adjudicate-exclusions run; judge + and install-exclusions do not need it (no prompt renders an exclusion).""" + sha = release_sha() + monkeypatch.setattr(driver, "release_exclusions_sha256", lambda: sha) + calls = [] + monkeypatch.setattr(driver, called, lambda *a: calls.append(called)) + stage = workspace / "results/local/stage" + installed = {"sha256": sha} if step in ("triage", "adjudicate-exclusions") else None + _resumable(stage, installed) + driver.main(["--stage-dir", str(stage), "--step", step]) + assert calls == [called] + + +# --- Reference pins ------------------------------------------------------------ + + +def test_the_committed_references_match_their_pins(tmp_path): + """Release 20261010's references, read from BASE_COMMIT, are the pinned + bytes: 58 exclusions (42 unlisted inputs, 14 engine defects and two + later-published law), and 1,926 scored outputs.""" + snapshot = base_references(tmp_path / "snapshot") + driver.verify_reference_pins(snapshot, "committed reference") + for name, pin in driver.BASE_REFERENCE_SHA256.items(): + raw = (snapshot / name).read_bytes() + assert hashlib.sha256(raw).hexdigest() == pin + from policybench.reference_exclusions import ( + load_reference_exclusions, + split_reference, + ) + + exclusions = load_reference_exclusions(snapshot) + assert len(exclusions) == driver.BASE_EXCLUSIONS + reasons = [e["reason_code"] for e in exclusions] + assert reasons.count("reference_engine_defect") == 14 + assert reasons.count("reference_depends_on_unlisted_input") == 42 + assert reasons.count("reference_law_published_after_freeze") == 2 + reference = pd.read_csv(snapshot / "reference_outputs.csv") + assert len(reference) == driver.BASE_OUTPUTS + assert reference.scenario_id.nunique() == 100 + assert len(split_reference(reference, exclusions)[0]) == driver.BASE_SCORED + + +def test_the_committed_adjudications_exclude_exactly_the_scoring_exclusions( + tmp_path, +): + """Release 20261010's record (70 decisions, 58 of them exclusions) excludes + exactly its exclusion record's outputs, scenario_023 Medicaid included.""" + from policybench.adjudications import excluded_case_keys + from policybench.reference_exclusions import ( + exclusion_keys, + load_reference_exclusions, + ) + + excluded = exclusion_keys( + load_reference_exclusions(base_references(tmp_path / "snapshot")) + ) + decisions = driver.base_adjudications() + assert len(decisions) == 70 + assert excluded_case_keys(decisions) == excluded + assert ("scenario_023", "head_medicaid_eligible") in excluded + + +@pytest.fixture +def pinned_copy(tmp_path): + """A copy of release 20261010's five reference files.""" + return base_references(tmp_path / "references") + + +@pytest.mark.parametrize("name", driver.REFERENCE_FILES) +@pytest.mark.parametrize("defect", ["appended", "missing"]) +def test_a_reference_file_that_misses_its_pin_is_refused(pinned_copy, name, defect): + driver.verify_reference_pins(pinned_copy, "staged reference") + if defect == "missing": + (pinned_copy / name).unlink() + else: + with (pinned_copy / name).open("a") as stream: + stream.write("\n") + with pytest.raises(SystemExit, match=f"staged reference {name} does not match"): + driver.verify_reference_pins(pinned_copy, "staged reference") + + +def test_the_release_exclusion_pin_replaces_only_the_exclusion_records(pinned_copy): + """The release rules nothing, so its exclusion pin is release 20261010's + own: the committed copy passes under it, and an edited record does not.""" + assert release_sha() == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + driver.verify_reference_pins(pinned_copy, "staged reference", release_sha()) + with (pinned_copy / EXCLUSIONS_NAME).open("a") as stream: + stream.write(" ") + with pytest.raises(SystemExit, match="reference_exclusions.json does not match"): + driver.verify_reference_pins(pinned_copy, "staged reference", release_sha()) + + +def test_resolve_base_checks_the_committed_pins_before_anything_else( + pinned_copy, monkeypatch +): + (pinned_copy / EXCLUSIONS_NAME).write_text('{"exclusions": []}\n') + monkeypatch.setattr(driver, "SNAPSHOT", pinned_copy) + monkeypatch.setattr( + driver, + "live_pointer", + lambda: pytest.fail("resolve_base read the pointer before the pins"), + ) + with pytest.raises( + SystemExit, match="committed reference reference_exclusions.json" + ): + driver.resolve_base(SimpleNamespace(partial=False)) + + +# --- Base payload in git ------------------------------------------------------- + + +def test_the_base_commit_holds_release_20261010(): + pointer = json.loads(driver.base_commit_blob(Path("app/src/data.artifact.json"))) + assert pointer["tag"] == driver.BASE_TAG + assert pointer["sha256"] == driver.BASE_SHA256 + for name, pin in driver.BASE_REFERENCE_SHA256.items(): + raw = driver.base_commit_blob(driver.SNAPSHOT.relative_to(driver.ROOT) / name) + assert hashlib.sha256(raw).hexdigest() == pin + + +def test_the_base_exclusion_record_is_read_from_git_and_pinned(monkeypatch): + assert driver.base_exclusion_record() == base_record() + # The scope check writes the record back with exclusions_text, so that + # spelling must reproduce the pinned bytes exactly. + assert ( + hashlib.sha256(driver.exclusions_text(base_record()).encode()).hexdigest() + == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + ) + monkeypatch.setattr( + driver, "base_commit_blob", lambda path: base_blob(path) + b"\n" + ) + with pytest.raises(SystemExit, match="does not match its pin"): + driver.base_exclusion_record() + + +def test_the_base_predictions_pin_is_the_release_commits_not_the_working_trees( + tmp_path, monkeypatch +): + """A working tree whose predictions and manifest pin are both replaced. + + The pin comes from the manifest at BASE_COMMIT, so editing the working + tree's manifest to match substituted predictions does not pass. + """ + committed = Path("paper/snapshot/20260501/manifest.json") + snapshot = tmp_path / "paper/snapshot/20260501/runs" / driver.RUN_NAME + snapshot.mkdir(parents=True) + substitute = snapshot / "predictions.csv.gz" + with gzip.open(substitute, "wt") as stream: + stream.write("model,scenario_id,variable,prediction\n") + manifest = json.loads(base_blob(committed)) + files = manifest["source_run_artifacts"][driver.RUN_NAME]["files"] + files["predictions.csv.gz"] = driver.digest(substitute) + (tmp_path / committed).write_text(json.dumps(manifest)) + monkeypatch.setattr(driver, "ROOT", tmp_path) + monkeypatch.setattr(driver, "base_commit_blob", base_blob) + with pytest.raises(SystemExit, match="fail release 20261010's manifest hash"): + driver.verify_base_predictions(snapshot) + + +def test_release_20261010s_manifest_pins_its_predictions(): + manifest = json.loads( + driver.base_commit_blob(Path("paper/snapshot/20260501/manifest.json")) + ) + pin = manifest["source_run_artifacts"][driver.RUN_NAME]["files"] + raw = driver.base_commit_blob( + driver.SNAPSHOT.relative_to(driver.ROOT) / "predictions.csv.gz" + ) + assert hashlib.sha256(raw).hexdigest() == pin["predictions.csv.gz"] + + +def test_base_commit_blob_names_the_missing_history(): + with pytest.raises(SystemExit, match="cannot read .* at base commit 5a8164a001ef"): + driver.base_commit_blob(Path("no/such/file.csv")) + + +def test_base_commit_blob_names_a_missing_commit(monkeypatch): + monkeypatch.setattr(driver, "BASE_COMMIT", "0" * 40) + with pytest.raises(SystemExit, match="at base commit 000000000000.*unshallow"): + driver.base_commit_blob(Path("app/src/data.artifact.json")) + + +def test_a_re_export_after_the_freeze_reads_20261010_from_git(monkeypatch): + """After the freeze the pointer names RELEASE_TAG; the base comes from git.""" + monkeypatch.setattr(driver, "live_pointer", lambda: {"tag": driver.RELEASE_TAG}) + monkeypatch.setattr( + driver, "resolve_base", lambda args: pytest.fail("read the working tree") + ) + live = driver.resolve_live_base(SimpleNamespace()) + names = {row["model"] for row in live["countries"]["us"]["modelStats"]} + assert len(names) == driver.BASE_MODELS and NEW not in names + + +def _gz_payload(models: int) -> bytes: + stats = [{"model": f"m{i}"} for i in range(models)] + return gzip.compress(json.dumps({"modelStats": stats}).encode()) + + +def test_the_git_base_must_rewrap_to_the_20261010_asset(monkeypatch): + raw = _gz_payload(47) + monkeypatch.setattr(driver, "base_commit_blob", lambda path: raw) + monkeypatch.setattr(driver, "live_pointer", lambda: {"tag": driver.RELEASE_TAG}) + with pytest.raises(SystemExit, match="base payload SHA256 mismatch"): + driver.resolve_live_base(SimpleNamespace()) + wrapped = json.dumps({"countries": {"us": json.loads(gzip.decompress(raw))}}) + monkeypatch.setattr( + driver, "BASE_SHA256", hashlib.sha256(wrapped.encode()).hexdigest() + ) + live = driver.resolve_live_base(SimpleNamespace()) + assert len(live["countries"]["us"]["modelStats"]) == 47 + short = _gz_payload(46) + monkeypatch.setattr(driver, "base_commit_blob", lambda path: short) + wrapped = json.dumps({"countries": {"us": json.loads(gzip.decompress(short))}}) + monkeypatch.setattr( + driver, "BASE_SHA256", hashlib.sha256(wrapped.encode()).hexdigest() + ) + with pytest.raises(SystemExit, match="base must have 47 models"): + driver.resolve_live_base(SimpleNamespace()) + + +@pytest.mark.parametrize("tag", ["dashboard-data-20260930", "dashboard-data-20991231"]) +def test_any_other_pointer_is_refused(monkeypatch, tag): + monkeypatch.setattr(driver, "live_pointer", lambda: {"tag": tag}) + with pytest.raises(SystemExit, match="base pointer changed"): + driver.resolve_live_base(SimpleNamespace()) + + +# --- Audit --------------------------------------------------------------------- + + +def _verdict(models, source="llm_error"): + return { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": source, + "case_failure_subtype": "thresholds_rates", + "rationale": "The models applied an outdated threshold.", + "models": [ + { + "model": model, + "failure_source": source, + "failure_subtype": "thresholds_rates", + "diagnosis": "The model used the prior year's threshold.", + } + for model in models + ], + } + + +def write_verdict(case, verdict, **meta): + """A verdict and its sidecar, bound to the verdict and the case's prompt; + a meta value of None leaves that key out.""" + path = case / "verdict.json" + path.write_text(json.dumps(verdict)) + prompt = case / "prompt.md" + sidecar = { + "verdict_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "prompt_sha256": ( + hashlib.sha256(prompt.read_bytes()).hexdigest() + if prompt.is_file() + else None + ), + "judge_runner": "scripts/run_audit_claude.sh", + "judge_model_requested": driver.JUDGE_MODEL, + "judge_model_reported": [driver.JUDGE_MODEL], + "judged_at_utc": "2026-10-09T01:00:00+00:00", + # A new verdict is on this release's template; a seed's sidecar + # records none (v1) and passes judge_template_version=None. + "judge_template_version": driver.JUDGE_TEMPLATE_VERSION, + **meta, + } + (case / "verdict.meta.json").write_text( + json.dumps({k: v for k, v in sidecar.items() if v is not None}) + ) + + +def _audit(root, wrong_models): + case = root / "cases/us__scenario_000__snap" + case.mkdir(parents=True) + (case / "prompt.md").write_text("Classify these wrong answers.\n") + (root / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA)) + (root / "cases.jsonl").write_text( + json.dumps( + { + "case_id": case.name, + "wrong_models": wrong_models, + "parse_failure_only": False, + } + ) + + "\n" + ) + return case + + +@pytest.fixture +def audit(tmp_path): + root = tmp_path / "audit" + case = _audit(root, [NEW]) + verdict = _verdict([NEW]) + write_verdict(case, verdict) + return root, case, verdict + + +def test_verdict_requires_full_schema_and_exact_wrong_model_coverage(audit): + root, _, _ = audit + assert driver.validate_verdicts(root) == [] + + +@pytest.mark.parametrize("defect", ["missing", "extra", "duplicate", "schema", "json"]) +def test_invalid_verdict_is_pending_and_removal_clears_provenance(audit, defect): + root, case, original = audit + verdict = copy.deepcopy(original) + if defect == "missing": + verdict["models"] = [] + elif defect == "extra": + verdict["models"].append({**verdict["models"][0], "model": "other"}) + elif defect == "duplicate": + verdict["models"].append(verdict["models"][0]) + elif defect == "schema": + del verdict["reference_suspect"] + write_verdict(case, verdict) + if defect == "json": + (case / "verdict.json").write_text("not JSON") + assert driver.validate_verdicts(root) == [case.name] + assert (case / "verdict.json").exists() + assert driver.validate_verdicts(root, remove_invalid=True) == [case.name] + assert not (case / "verdict.json").exists() + assert not (case / "verdict.meta.json").exists() + + +@pytest.mark.parametrize( + "defect", + [ + "missing", + "stale", + "wrong_judge", + "not_an_object", + "no_timestamp", + "no_prompt_sha256", + "other_prompt", + "no_runner", + "requested_other", + ], +) +def test_new_model_verdict_requires_bound_opus55_provenance(audit, defect): + root, case, _ = audit + path = case / "verdict.meta.json" + meta = json.loads(path.read_text()) + if defect == "missing": + path.unlink() + else: + if defect == "stale": + meta["verdict_sha256"] = "0" * 64 + elif defect == "wrong_judge": + meta["judge_model_reported"] = ["claude-opus-5"] + elif defect == "no_timestamp": + del meta["judged_at_utc"] + elif defect == "no_prompt_sha256": + del meta["prompt_sha256"] + elif defect == "other_prompt": + meta["prompt_sha256"] = hashlib.sha256(b"another prompt").hexdigest() + elif defect == "no_runner": + del meta["judge_runner"] + elif defect == "requested_other": + meta["judge_model_requested"] = "claude-sonnet-5-5" + else: + meta = [meta] + path.write_text(json.dumps(meta)) + assert driver.validate_verdicts(root) == [case.name] + + +def test_every_verdict_is_bound_to_its_own_bytes(tmp_path): + """An incumbent verdict's sidecar must match its bytes too, not only a + verdict that names Muse Wonder.""" + root = tmp_path / "audit" + case = _audit(root, ["incumbent"]) + write_verdict( + case, + _verdict(["incumbent"]), + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + ) + assert driver.validate_verdicts(root) == [] + edited = _verdict(["incumbent"]) + edited["case_failure_subtype"] = "other" + (case / "verdict.json").write_text(json.dumps(edited)) + assert driver.validate_verdicts(root) == [case.name] + + +def test_a_verdict_stays_bound_to_its_prompt_sha256(tmp_path): + root = tmp_path / "audit" + case = _audit(root, ["incumbent"]) + prompt = hashlib.sha256((case / "prompt.md").read_bytes()).hexdigest() + write_verdict( + case, + _verdict(["incumbent"]), + prompt_sha256=prompt, + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + ) + assert driver.validate_verdicts(root) == [] + (case / "prompt.md").write_text("Classify these wrong answers, and one more.\n") + assert driver.validate_verdicts(root, remove_invalid=True) == [case.name] + assert not (case / "verdict.json").exists() + # Set aside, not destroyed. + (kept,) = (tmp_path / "rejected-verdicts" / case.name).iterdir() + assert sorted(p.name for p in kept.iterdir()) == [ + "reason.txt", + "verdict.json", + "verdict.meta.json", + ] + + +def test_the_judge_must_cover_all_47_models_in_a_case(tmp_path): + incumbents = [f"incumbent-{i:02d}" for i in range(46)] + root = tmp_path / "audit" + case = _audit(root, [*incumbents, NEW]) + write_verdict(case, _verdict(incumbents)) + assert driver.validate_verdicts(root) == [case.name] + write_verdict(case, _verdict([*incumbents, NEW])) + assert driver.validate_verdicts(root) == [] + # A carried-over incumbent verdict cannot name the new model, and a + # re-judge from another judge does not qualify. + write_verdict( + case, + _verdict([*incumbents, NEW]), + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + ) + assert driver.validate_verdicts(root) == [case.name] + + +def test_parse_only_cases_do_not_require_paid_judging(audit): + root, case, _ = audit + manifest = json.loads((root / "cases.jsonl").read_text()) + manifest["parse_failure_only"] = True + (root / "cases.jsonl").write_text(json.dumps(manifest) + "\n") + (case / "verdict.json").unlink() + (case / "verdict.meta.json").unlink() + assert driver.validate_verdicts(root) == [] + + +def test_unchanged_incumbent_case_keeps_its_existing_judge(tmp_path): + """A seed verdict whose sidecar never recorded its prompt carries over + through the seed binding.""" + root = tmp_path / "audit" + case = _audit(root, ["incumbent"]) + write_verdict( + case, + _verdict(["incumbent"]), + judge_model_requested="default", + judge_model_reported=["gpt-5.6-sol"], + judge_runner="scripts/run_audit_codex.sh", + prompt_sha256=None, + ) + seed = driver.seed_digest(root) + assert driver.validate_verdicts(root, seed=seed) == [] + # Without the binding nothing ties the verdict to its prompt. + assert driver.validate_verdicts(root) == [case.name] + + +@pytest.mark.parametrize("mutation", ["edited", "rebound", "missing"]) +def test_a_carried_over_verdict_must_keep_the_seeds_bytes(tmp_path, mutation): + """Editing a carried-over verdict, even with its sidecar re-hashed to + match, or removing it, is refused: a re-judge cannot restore it.""" + root = tmp_path / "audit" + case = _audit(root, ["incumbent"]) + write_verdict( + case, + _verdict(["incumbent"]), + judge_model_requested="default", + judge_model_reported=["gpt-5.6-sol"], + prompt_sha256=None, + ) + seed = driver.seed_digest(root) + edited = _verdict(["incumbent"]) + edited["case_failure_subtype"] = "other" + if mutation == "edited": + (case / "verdict.json").write_text(json.dumps(edited)) + elif mutation == "rebound": + write_verdict( + case, + edited, + judge_model_requested="default", + judge_model_reported=["gpt-5.6-sol"], + ) + else: + (case / "verdict.json").unlink() + before = sorted(p.name for p in case.iterdir()) + with pytest.raises(SystemExit, match="carried-over verdicts differ"): + driver.validate_verdicts(root, remove_invalid=True, seed=seed) + assert sorted(p.name for p in case.iterdir()) == before + + +# A two-household board whose audit prompts render for real (policybench.audit). +REFERENCE_ROWS = [ + {"scenario_id": "s0", "variable": "snap", "value": 0.0}, + {"scenario_id": "s1", "variable": "snap", "value": 300.0}, + {"scenario_id": "s2", "variable": "snap", "value": 100.0}, +] +INCUMBENT_ROWS = [ + ("m1", "s0", 250.0, "Estimated benefit from income."), + ("m2", "s0", 400.0, "Used gross income test."), + ("m1", "s1", 300.0, "Matched the allotment."), + ("m2", "s1", 0.0, "Assumed ineligible."), + ("m1", "s2", 100.0, "Matched the allotment."), + ("m2", "s2", 100.0, "Matched the allotment."), +] + + +def _board(directory: Path, rows) -> Path: + us = directory / "us" + us.mkdir(parents=True) + pd.DataFrame(REFERENCE_ROWS).to_csv(us / "reference_outputs.csv", index=False) + pd.DataFrame( + [ + { + "model": model, + "scenario_id": scenario, + "variable": "snap", + "prediction": value, + "explanation": text, + "error": None, + } + for model, scenario, value, text in rows + ] + ).to_csv(us / "predictions.csv", index=False) + return directory + + +@pytest.fixture +def seeded_stage(tmp_path, monkeypatch): + """A judged 2-model seed audit and a stage whose board adds the new model. + + No reference explanation is reworded unless a test says so: the real + reworded_since_seed() names cases of the real board, which this one lacks. + """ + from policybench.audit import prepare_audit + + monkeypatch.setattr(driver, "reworded_since_seed", lambda: frozenset()) + grounding = tmp_path / "grounding.csv" + pd.DataFrame( + [{"scenario_id": "s1", "variable": "snap", "grounding": "Gross test: pass."}] + ).to_csv(grounding, index=False) + monkeypatch.setattr( + driver, "GROUNDING_SHA256", hashlib.sha256(grounding.read_bytes()).hexdigest() + ) + lookup = {("s1", "snap"): "Gross test: pass."} + seed = tmp_path / "seed" + seed_board = _board(tmp_path / "seed-board", INCUMBENT_ROWS) + # Release 20261010's seed was judged on judge template v1, before versions + # existed, so its sidecars record none. + prepare_audit(seed_board / "us", seed, grounding_lookup=lookup, template_version=1) + for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): + write_verdict( + seed / "cases" / item["case_id"], + _verdict(item["wrong_models"]), + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + prompt_sha256=None, + judge_template_version=None, + ) + digest_text = driver.seed_digest_text(driver.seed_digest(seed)) + monkeypatch.setattr( + driver, "SEED_DIGEST_SHA256", hashlib.sha256(digest_text.encode()).hexdigest() + ) + stage = tmp_path / "stage" + + def prepare(new_rows, *, grounding_path=grounding, explanations=None): + bundle = _board(stage / "publish" / driver.RUN_NAME, INCUMBENT_ROWS + new_rows) + if explanations: + # The one annotation a judge's prompt carries: rewording it + # re-renders the case's prompt, as release 20261010's rewording did. + (bundle / "annotations").mkdir() + pd.DataFrame( + [ + {"scenario_id": scenario, "variable": "snap", "explanation": text} + for scenario, text in explanations.items() + ] + ).to_csv( + bundle / "annotations/us_case_reference_explanations.csv", index=False + ) + args = SimpleNamespace( + stage_dir=stage, audit_seed=seed, grounding=grounding_path + ) + binding = driver.prepare_cases(args, bundle) + # main binds the seed in stage.json beside the input hashes. + (stage / "stage.json").write_text( + json.dumps({"partial": False, "early": False, "files": {}, "seed": binding}) + ) + return stage / "audit" + + return seed, stage, prepare + + +def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( + seeded_stage, +): + seed, stage, prepare = seeded_stage + audit = prepare( + [ + (NEW, "s0", 99.0, "Guessed."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", 100.0, "Right."), + ] + ) + joined, untouched = "us__s0__snap", "us__s1__snap" + assert json.loads((stage / "pending.json").read_text()) == [joined] + assert not (audit / "cases" / joined / "verdict.json").exists() + assert not (audit / "cases" / joined / "verdict.meta.json").exists() + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + assert (audit / "cases" / untouched / name).read_bytes() == ( + seed / "cases" / untouched / name + ).read_bytes() + assert json.loads((stage / "prompt-changes.json").read_text()) == { + "added": [], + "changed": [joined], + "kept": [untouched], + } + wrong = ["m1", "m2", NEW] + write_verdict(audit / "cases" / joined, _verdict(wrong)) + seed_binding = driver.load_seed(stage) + assert seed_binding == driver.seed_digest(seed) + assert driver.validate_verdicts(audit, seed=seed_binding) == [] + + +# Muse Wonder misses s0 alone: s0 is re-opened, s1 carries over. +JOINS_S0 = [ + (NEW, "s0", 99.0, "Guessed."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", 100.0, "Right."), +] +# Muse Wonder answers every output right: no case is joined. +JOINS_NONE = [ + (NEW, "s0", 0.0, "Ineligible."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", 100.0, "Right."), +] + + +def test_an_edited_carried_over_verdict_is_refused_after_prepare(seeded_stage): + """Edit a kept incumbent verdict in a prepared stage (sidecar re-hashed to + match) and validation refuses it.""" + _, stage, prepare = seeded_stage + audit = prepare(JOINS_S0) + write_verdict(audit / "cases/us__s0__snap", _verdict(["m1", "m2", NEW])) + seed = driver.load_seed(stage) + assert driver.validate_verdicts(audit, seed=seed) == [] + kept = audit / "cases/us__s1__snap" + verdict = json.loads((kept / "verdict.json").read_text()) + verdict["case_failure_source"] = "prompt_ambiguity" + write_verdict( + kept, + verdict, + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + ) + with pytest.raises(SystemExit, match="carried-over verdicts differ.*us__s1__snap"): + driver.validate_verdicts(audit, seed=seed) + + +def test_prepare_refuses_a_seed_the_committed_digest_does_not_record( + seeded_stage, monkeypatch +): + _, stage, prepare = seeded_stage + monkeypatch.setattr(driver, "SEED_DIGEST_SHA256", COMMITTED_SEED_DIGEST_SHA256) + with pytest.raises(SystemExit, match="not release 20261010's"): + prepare(JOINS_S0) + assert not (stage / "audit").exists() + + +def test_load_seed_refuses_a_stage_without_a_binding_or_with_another(seeded_stage): + _, stage, prepare = seeded_stage + prepare(JOINS_S0) + receipt = json.loads((stage / "stage.json").read_text()) + edited = copy.deepcopy(receipt) + edited["seed"]["us__s1__snap"]["verdict_sha256"] = "0" * 64 + (stage / "stage.json").write_text(json.dumps(edited)) + with pytest.raises(SystemExit, match="not release 20261010's"): + driver.load_seed(stage) + del receipt["seed"] + (stage / "stage.json").write_text(json.dumps(receipt)) + with pytest.raises(SystemExit, match="bind-seed"): + driver.load_seed(stage) + + +def test_bind_seed_binds_a_stage_prepared_before_prepare_did(seeded_stage): + seed, stage, prepare = seeded_stage + prepare(JOINS_S0) + assert json.loads((stage / driver.PROMPT_CHANGES).read_text())["kept"] == [ + "us__s1__snap" + ] + receipt = json.loads((stage / "stage.json").read_text()) + binding = receipt.pop("seed") + (stage / "stage.json").write_text(json.dumps(receipt)) + args = SimpleNamespace(stage_dir=stage, audit_seed=seed) + driver.bind_seed(args) + assert driver.load_seed(stage) == binding + with pytest.raises(SystemExit, match="already binds"): + driver.bind_seed(args) + + +@pytest.mark.parametrize("move", ["kept_to_changed", "changed_to_kept", "dropped"]) +def test_prompt_changes_that_disagree_with_the_stage_stop_every_step( + seeded_stage, move +): + """prompt-changes.json is re-derived from the stage's prompts and the + bound seed on every read: moving a case between the lists, or dropping + one, is refused.""" + _, stage, prepare = seeded_stage + prepare(JOINS_S0) + path = stage / driver.PROMPT_CHANGES + changes = json.loads(path.read_text()) + if move == "kept_to_changed": + changes["kept"].remove("us__s1__snap") + changes["changed"].append("us__s1__snap") + elif move == "changed_to_kept": + changes["changed"].remove("us__s0__snap") + changes["kept"].append("us__s0__snap") + else: + changes["changed"].remove("us__s0__snap") + path.write_text(json.dumps(changes)) + for step in (driver.load_seed, driver.rejudged_cases): + with pytest.raises(SystemExit, match="disagrees with the stage's prompts"): + step(stage) + + +@pytest.mark.parametrize("edit", ["prompt_only", "verdict_sidecar_and_prompt"]) +def test_a_kept_case_cannot_pass_as_a_new_verdict(seeded_stage, edit): + """A kept case whose prompt is edited (with or without a re-bound + verdict) is refused, not re-judged.""" + _, stage, prepare = seeded_stage + audit = prepare(JOINS_S0) + write_verdict(audit / "cases/us__s0__snap", _verdict(["m1", "m2", NEW])) + kept = audit / "cases/us__s1__snap" + (kept / "prompt.md").write_text((kept / "prompt.md").read_text() + "\n") + if edit == "verdict_sidecar_and_prompt": + verdict = json.loads((kept / "verdict.json").read_text()) + verdict["case_failure_subtype"] = "other" + write_verdict(kept, verdict) + before = sorted(p.name for p in kept.iterdir()) + with pytest.raises(SystemExit, match="incumbent-only case prompts changed"): + driver.load_seed(stage) + assert sorted(p.name for p in kept.iterdir()) == before + + +@pytest.mark.parametrize("claim", ["reopens_a_right_answer", "keeps_a_wrong_answer"]) +def test_the_manifest_cannot_reopen_a_case_the_predictions_do_not(seeded_stage, claim): + """Muse Wonder answers s1 right and s0 wrong. Listing it as wrong on + s1 (prompt edited, case moved to changed), or dropping it from s0 (seed + prompt restored, case moved to kept), leaves the re-derived lists + agreeing, so only the predictions can refuse it.""" + seed, stage, prepare = seeded_stage + audit = prepare(JOINS_S0) + driver.load_seed(stage) + manifest = [ + json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() + ] + changes = json.loads((stage / driver.PROMPT_CHANGES).read_text()) + case = "us__s1__snap" if claim == "reopens_a_right_answer" else "us__s0__snap" + row = next(item for item in manifest if item["case_id"] == case) + if claim == "reopens_a_right_answer": + row["wrong_models"].append(NEW) + prompt = audit / "cases" / case / "prompt.md" + prompt.write_text(prompt.read_text() + f"{NEW}: Wrong.\n") + changes["kept"].remove(case) + changes["changed"].append(case) + else: + row["wrong_models"].remove(NEW) + shutil.copyfile( + seed / "cases" / case / "prompt.md", audit / "cases" / case / "prompt.md" + ) + changes["changed"].remove(case) + changes["kept"].append(case) + (audit / "cases.jsonl").write_text("".join(json.dumps(r) + "\n" for r in manifest)) + (stage / driver.PROMPT_CHANGES).write_text(json.dumps(changes)) + for step in (driver.load_seed, driver.rejudged_cases): + with pytest.raises( + SystemExit, match=f"disagree with the staged predictions.*{case}" + ): + step(stage) + + +def test_set_aside_keeps_the_judge_evidence_with_the_verdict(tmp_path): + root = tmp_path / "audit" + case = _audit(root, [NEW]) + write_verdict(case, _verdict(["someone-else"])) + for name in ("claude.json", "claude.log", "claude.transcript.jsonl"): + (case / name).write_text(name) + assert driver.validate_verdicts(root, remove_invalid=True) == [case.name] + (kept,) = (tmp_path / "rejected-verdicts" / case.name).iterdir() + assert sorted(p.name for p in kept.iterdir()) == [ + "claude.json", + "claude.log", + "claude.transcript.jsonl", + "reason.txt", + "verdict.json", + "verdict.meta.json", + ] + assert sorted(p.name for p in case.iterdir()) == ["prompt.md"] + + +@pytest.mark.parametrize("payload", [[], "text", {"amendments": {}}]) +def test_a_malformed_amendment_file_is_refused(tmp_path, payload): + (tmp_path / driver.AMENDMENTS).write_text(json.dumps(payload)) + with pytest.raises(SystemExit, match="'amendments' is not a list"): + driver.load_amendments(tmp_path, frozenset()) + + +@pytest.mark.parametrize("defect", ["kept_verdict", "kept_prompt", "changed_prompt"]) +def test_bind_seed_refuses_a_stage_that_disagrees_with_the_seed(seeded_stage, defect): + seed, stage, prepare = seeded_stage + audit = prepare(JOINS_S0) + receipt = json.loads((stage / "stage.json").read_text()) + del receipt["seed"] + (stage / "stage.json").write_text(json.dumps(receipt)) + if defect == "kept_verdict": + (audit / "cases/us__s1__snap/verdict.json").write_text("{}") + elif defect == "kept_prompt": + (audit / "cases/us__s1__snap/prompt.md").write_text("Another prompt.\n") + else: + shutil.copyfile( + seed / "cases/us__s0__snap/prompt.md", + audit / "cases/us__s0__snap/prompt.md", + ) + with pytest.raises(SystemExit, match="disagrees with the seed"): + driver.bind_seed(SimpleNamespace(stage_dir=stage, audit_seed=seed)) + assert "seed" not in json.loads((stage / "stage.json").read_text()) + + +def test_a_household_only_the_new_model_misses_becomes_a_new_case(seeded_stage): + _, stage, prepare = seeded_stage + audit = prepare( + [ + (NEW, "s0", 0.0, "Ineligible."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", 55.0, "Halved."), + ] + ) + assert json.loads((stage / "prompt-changes.json").read_text()) == { + "added": ["us__s2__snap"], + "changed": [], + "kept": ["us__s0__snap", "us__s1__snap"], + } + assert json.loads((stage / "pending.json").read_text()) == ["us__s2__snap"] + assert not (audit / "cases/us__s2__snap/verdict.json").exists() + + +def test_an_incumbent_prompt_that_changes_is_refused( + seeded_stage, tmp_path, monkeypatch +): + _, _, prepare = seeded_stage + other = tmp_path / "other-grounding.csv" + pd.DataFrame( + [{"scenario_id": "s1", "variable": "snap", "grounding": "Gross test: fail."}] + ).to_csv(other, index=False) + monkeypatch.setattr( + driver, "GROUNDING_SHA256", hashlib.sha256(other.read_bytes()).hexdigest() + ) + with pytest.raises(SystemExit, match=r"incumbent-only case prompts changed.*s1"): + prepare(JOINS_S0, grounding_path=other) + + +def test_prepare_refuses_a_grounding_other_than_the_pinned_one(seeded_stage, tmp_path): + _, stage, prepare = seeded_stage + other = tmp_path / "other-grounding.csv" + other.write_text("scenario_id,variable,grounding\n") + with pytest.raises(SystemExit, match="grounding differs"): + prepare([], grounding_path=other) + assert not (stage / "audit").exists() + + +def test_the_pinned_grounding_is_the_one_the_seed_was_rendered_with(): + if not GROUNDING.is_file(): + pytest.skip("the main clone's audit grounding is not on this machine") + assert driver.digest(GROUNDING) == driver.GROUNDING_SHA256 + + +# --- The nine reworded cases --------------------------------------------------- + +REWORDED_S1 = {"s1": "The allotment is the maximum less 30% of net income."} + + +def test_a_reworded_case_may_change_without_the_new_model(seeded_stage, monkeypatch): + """s1's reference explanation is reworded and Muse Wonder answers it + right: its prompt changes, it is re-opened and re-judged, and every gate + that re-derives the lists agrees.""" + _, stage, prepare = seeded_stage + monkeypatch.setattr( + driver, "reworded_since_seed", lambda: frozenset({"us__s1__snap"}) + ) + audit = prepare(JOINS_S0, explanations=REWORDED_S1) + assert json.loads((stage / driver.PROMPT_CHANGES).read_text()) == { + "added": [], + "changed": ["us__s0__snap", "us__s1__snap"], + "kept": [], + } + assert json.loads((stage / "pending.json").read_text()) == [ + "us__s0__snap", + "us__s1__snap", + ] + assert ( + "HOW POLICYENGINE DERIVED" + in (audit / "cases/us__s1__snap/prompt.md").read_text() + ) + manifest = { + item["case_id"]: item + for item in map(json.loads, (audit / "cases.jsonl").read_text().splitlines()) + } + assert NEW not in manifest["us__s1__snap"]["wrong_models"] + assert driver.rejudged_cases(stage) == {"us__s0__snap", "us__s1__snap"} + write_verdict(audit / "cases/us__s0__snap", _verdict(["m1", "m2", NEW])) + write_verdict(audit / "cases/us__s1__snap", _verdict(["m2"])) + assert driver.validate_verdicts(audit, seed=driver.load_seed(stage)) == [] + + +def test_a_reworded_case_the_new_model_joins_is_rejudged(seeded_stage, monkeypatch): + """The real stage's case: Muse Wonder answers all nine wrong.""" + _, stage, prepare = seeded_stage + monkeypatch.setattr( + driver, "reworded_since_seed", lambda: frozenset({"us__s0__snap"}) + ) + prepare(JOINS_S0, explanations={"s0": "The household fails the net test."}) + assert json.loads((stage / driver.PROMPT_CHANGES).read_text()) == { + "added": [], + "changed": ["us__s0__snap"], + "kept": ["us__s1__snap"], + } + + +def test_a_reworded_explanation_outside_the_pinned_set_is_refused(seeded_stage): + _, stage, prepare = seeded_stage + with pytest.raises( + SystemExit, match=r"incumbent-only case prompts changed.*\['us__s1__snap'\]" + ): + prepare(JOINS_S0, explanations=REWORDED_S1) + + +def test_the_reworded_allowance_admits_no_other_incumbent_only_change( + seeded_stage, monkeypatch +): + """s0 and s1 both change without the new model; only s1 is reworded, so + the refusal names s0 alone.""" + _, _, prepare = seeded_stage + monkeypatch.setattr( + driver, "reworded_since_seed", lambda: frozenset({"us__s1__snap"}) + ) + explanations = {**REWORDED_S1, "s0": "The household fails the net test."} + with pytest.raises( + SystemExit, match=r"incumbent-only case prompts changed.*: \['us__s0__snap'\]$" + ): + prepare(JOINS_NONE, explanations=explanations) + + +@pytest.mark.parametrize("reworded", ["us__s1__snap", "us__s9__snap"]) +def test_a_reworded_case_whose_prompt_did_not_change_is_refused( + seeded_stage, monkeypatch, reworded +): + """A reworded case kept byte for byte (or absent from the audit) means + the rewording did not reach its prompt: prepare stops.""" + _, _, prepare = seeded_stage + monkeypatch.setattr(driver, "reworded_since_seed", lambda: frozenset({reworded})) + with pytest.raises( + SystemExit, + match=re.escape(f"reworded cases whose prompts did not change: {[reworded]}"), + ): + prepare(JOINS_S0) + + +def test_check_prompt_changes_names_new_incumbent_only_cases(tmp_path, monkeypatch): + monkeypatch.setattr(driver, "reworded_since_seed", lambda: frozenset()) + root = tmp_path / "audit" + case = _audit(root, ["incumbent"]) + with pytest.raises(SystemExit, match=case.name): + driver.check_prompt_changes(root, {}) + manifest = json.loads((root / "cases.jsonl").read_text()) + manifest["wrong_models"] = ["incumbent", NEW] + (root / "cases.jsonl").write_text(json.dumps(manifest) + "\n") + assert driver.check_prompt_changes(root, {})["added"] == [case.name] + with pytest.raises(SystemExit, match="seed cases vanished.*us__s9__snap"): + driver.check_prompt_changes(root, {"us__s9__snap": "0" * 64}) + + +CASE_FLAGS = st.fixed_dictionaries( + { + "seeded": st.booleans(), + "same_prompt": st.booleans(), + "new_wrong": st.booleans(), + "reworded": st.booleans(), + "parse_only": st.booleans(), + } +) + + +@settings(max_examples=150, deadline=None) +@given( + flags=st.lists(CASE_FLAGS, max_size=7), + vanished=st.integers(min_value=0, max_value=2), + stray_reworded=st.booleans(), +) +def test_check_prompt_changes_follows_its_rule_for_any_audit( + flags, vanished, stray_reworded +): + """For any manifest, seed and reworded set: a judged case is kept when its + prompt is the seed's, changed when the seed has another, added when the + seed lacks it. It refuses, in this order, a changed or added case neither + the new model joins nor a rewording explains, a seed case that is not + kept or changed, and a reworded case that is not changed.""" + with tempfile.TemporaryDirectory() as scratch: + audit = Path(scratch) / "audit" + manifest, seeded, reworded = [], {}, set() + for index, case_flags in enumerate(flags): + case = f"us__s{index}__snap" + (audit / "cases" / case).mkdir(parents=True) + prompt = audit / "cases" / case / "prompt.md" + prompt.write_text(f"Prompt {index}.\n") + wrong = ["m1", *([NEW] if case_flags["new_wrong"] else [])] + manifest.append( + { + "case_id": case, + "wrong_models": wrong, + "parse_failure_only": case_flags["parse_only"], + } + ) + if case_flags["seeded"]: + seeded[case] = ( + driver.digest(prompt) + if case_flags["same_prompt"] + else hashlib.sha256(f"Seed prompt {index}.\n".encode()).hexdigest() + ) + if case_flags["reworded"]: + reworded.add(case) + for index in range(vanished): + seeded[f"us__gone{index}__snap"] = "0" * 64 + if stray_reworded: + reworded.add("us__elsewhere__snap") + audit.mkdir(parents=True, exist_ok=True) + (audit / "cases.jsonl").write_text( + "".join(json.dumps(item) + "\n" for item in manifest) + ) + judged = [ + (item, flags[i]) + for i, item in enumerate(manifest) + if not item["parse_failure_only"] + ] + kept = [ + item["case_id"] for item, f in judged if f["seeded"] and f["same_prompt"] + ] + changed = [ + item["case_id"] + for item, f in judged + if f["seeded"] and not f["same_prompt"] + ] + added = [item["case_id"] for item, f in judged if not f["seeded"]] + offending = [ + item["case_id"] + for item, f in judged + if item["case_id"] not in kept + and not f["new_wrong"] + and item["case_id"] not in reworded + ] + gone = sorted(set(seeded) - {*kept, *changed}) + unrendered = sorted(reworded - set(changed)) + with mock.patch.object( + driver, "reworded_since_seed", lambda: frozenset(reworded) + ): + if offending or gone or unrendered: + with pytest.raises(SystemExit) as refusal: + driver.check_prompt_changes(audit, seeded) + message = str(refusal.value) + if offending: + assert "incumbent-only case prompts changed" in message + assert message.endswith(repr(offending[:8])) + elif gone: + assert message == f"seed cases vanished from the audit: {gone[:8]}" + else: + assert message == ( + f"reworded cases whose prompts did not change: {unrendered[:8]}" + ) + else: + assert driver.check_prompt_changes(audit, seeded) == { + "kept": kept, + "changed": changed, + "added": added, + } + + +def test_the_reworded_cases_are_exactly_those_git_shows(): + """Runs anywhere with full history: the explanations committed at + SEED_RELEASE_COMMIT and BASE_COMMIT differ on exactly the nine pinned + cases. Read again here with the csv module, the files say the same.""" + import csv + import io + + assert driver.reworded_since_seed() == driver.REWORDED_SINCE_SEED + + def rows(commit: str) -> dict: + blob = subprocess.run( + [ + "git", + "-C", + str(REPO), + "show", + f"{commit}:{driver.REFERENCE_EXPLANATIONS.as_posix()}", + ], + capture_output=True, + check=True, + ).stdout + reader = csv.DictReader(io.StringIO(blob.decode("utf-8"))) + return {(row["scenario_id"], row["variable"]): row for row in reader} + + before, after = rows(driver.SEED_RELEASE_COMMIT), rows(BASE_COMMIT) + assert set(before) == set(after) + reworded = { + f"us__{scenario}__{variable}" + for (scenario, variable), row in after.items() + if before[(scenario, variable)] != row + } + assert reworded == driver.REWORDED_SINCE_SEED + # The rewording touches the explanation, the column a prompt renders. + for case in reworded: + _, scenario, variable = case.split("__", 2) + old, new = before[(scenario, variable)], after[(scenario, variable)] + assert old["explanation"] != new["explanation"] + + +def _explanations_csv(rows: list[dict]) -> bytes: + return ( + pd.DataFrame(rows, columns=["scenario_id", "variable", "explanation"]) + .to_csv(index=False) + .encode() + ) + + +EXPLANATION_ROWS = [ + {"scenario_id": f"scenario_00{i}", "variable": "snap", "explanation": text} + for i, text in enumerate( + ["Gross test passes.", 'Net income is $1,200, "after deductions".', "Zero."] + ) +] + + +@pytest.fixture +def explanation_repo(tmp_path, monkeypatch): + """A git repository that commits the explanations file twice, the seed's + and the base's, with the driver pointed at both commits.""" + root = tmp_path / "repo" + path = root / driver.REFERENCE_EXPLANATIONS + path.parent.mkdir(parents=True) + subprocess.run(["git", "-C", str(root), "init", "-q"], check=True) + monkeypatch.setattr(driver, "ROOT", root) + + def commits(before: list[dict], after: list[dict], pinned: set[str]): + path.write_bytes(_explanations_csv(before)) + monkeypatch.setattr(driver, "SEED_RELEASE_COMMIT", commit(root, "seed")) + path.write_bytes(_explanations_csv(after)) + monkeypatch.setattr(driver, "BASE_COMMIT", commit(root, "base")) + monkeypatch.setattr(driver, "REWORDED_SINCE_SEED", frozenset(pinned)) + + return commits + + +def _reword(rows: list[dict], *indices: int) -> list[dict]: + return [ + {**row, "explanation": row["explanation"] + " Reworded."} + if index in indices + else dict(row) + for index, row in enumerate(rows) + ] + + +def test_reworded_since_seed_reads_both_commits_from_git(explanation_repo): + explanation_repo( + EXPLANATION_ROWS, _reword(EXPLANATION_ROWS, 1), {"us__scenario_001__snap"} + ) + assert driver.reworded_since_seed() == {"us__scenario_001__snap"} + + +def test_reworded_since_seed_refuses_another_reworded_explanation(explanation_repo): + explanation_repo( + EXPLANATION_ROWS, _reword(EXPLANATION_ROWS, 1, 2), {"us__scenario_001__snap"} + ) + with pytest.raises( + SystemExit, match=re.escape("pinned set: ['us__scenario_002__snap']") + ): + driver.reworded_since_seed() + + +def test_reworded_since_seed_refuses_a_pinned_case_left_as_it_was(explanation_repo): + explanation_repo( + EXPLANATION_ROWS, + _reword(EXPLANATION_ROWS, 1), + {"us__scenario_000__snap", "us__scenario_001__snap"}, + ) + with pytest.raises( + SystemExit, match=re.escape("pinned set: ['us__scenario_000__snap']") + ): + driver.reworded_since_seed() + + +@pytest.mark.parametrize("change", ["added", "removed"]) +def test_reworded_since_seed_refuses_a_changed_set_of_cases(explanation_repo, change): + extra = {"scenario_id": "scenario_009", "variable": "snap", "explanation": "New."} + after = EXPLANATION_ROWS + [extra] if change == "added" else EXPLANATION_ROWS[:-1] + explanation_repo(EXPLANATION_ROWS, after, set()) + with pytest.raises(SystemExit, match="explanation keys changed since the seed"): + driver.reworded_since_seed() + + +def test_reworded_since_seed_names_a_commit_it_cannot_read( + explanation_repo, monkeypatch +): + explanation_repo(EXPLANATION_ROWS, EXPLANATION_ROWS, set()) + monkeypatch.setattr(driver, "SEED_RELEASE_COMMIT", "0" * 40) + with pytest.raises(SystemExit, match="cannot read .* at 000000000000"): + driver.reworded_since_seed() + + +def test_a_reordered_explanation_file_rewords_nothing(explanation_repo): + explanation_repo(EXPLANATION_ROWS, EXPLANATION_ROWS[::-1], set()) + assert driver.reworded_since_seed() == frozenset() + + +EXPLANATION_TEXT = st.text( + alphabet=st.characters( + blacklist_categories=("Cs", "Cc"), whitelist_characters="\n,\"' " + ), + max_size=30, +) + + +@settings(max_examples=100, deadline=None) +@given(texts=st.lists(EXPLANATION_TEXT, min_size=1, max_size=8), data=st.data()) +def test_reworded_since_seed_is_exactly_the_rows_that_differ(texts, data): + """With git stubbed: for any explanations and any reworded rows, the + derived set is those rows; it is returned when it is the pinned set and + refused, naming the difference, when it is not.""" + count = len(texts) + changed = data.draw(st.sets(st.integers(min_value=0, max_value=count - 1))) + pinned_rows = data.draw( + st.one_of( + st.just(changed), st.sets(st.integers(min_value=0, max_value=count - 1)) + ) + ) + before = [ + {"scenario_id": f"scenario_{i:03d}", "variable": "snap", "explanation": text} + for i, text in enumerate(texts) + ] + blobs = { + "seed-commit": _explanations_csv(before), + "base-commit": _explanations_csv(_reword(before, *changed)), + } + asked = [] + + def run(command, capture_output): + commit_id, path = command[-1].split(":", 1) + asked.append(path) + return SimpleNamespace(returncode=0, stdout=blobs[commit_id], stderr=b"") + + def cases(rows): + return frozenset(f"us__scenario_{i:03d}__snap" for i in rows) + + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch.object(driver, "subprocess", SimpleNamespace(run=run)) + ) + stack.enter_context( + mock.patch.object(driver, "SEED_RELEASE_COMMIT", "seed-commit") + ) + stack.enter_context(mock.patch.object(driver, "BASE_COMMIT", "base-commit")) + stack.enter_context( + mock.patch.object(driver, "REWORDED_SINCE_SEED", cases(pinned_rows)) + ) + if changed == pinned_rows: + assert driver.reworded_since_seed() == cases(changed) + else: + difference = sorted(cases(changed) ^ cases(pinned_rows)) + with pytest.raises(SystemExit, match=re.escape(str(difference))): + driver.reworded_since_seed() + assert asked == [str(driver.REFERENCE_EXPLANATIONS)] * 2 + + +# --- The committed seed -------------------------------------------------------- + + +def _committed_seed_digest() -> dict[str, dict[str, str]]: + lines = driver.SEED_DIGEST.read_text().splitlines() + assert lines[0] == "case_id,prompt_sha256,verdict_sha256" + rows = [line.split(",") for line in lines[1:]] + return { + case: {"prompt_sha256": prompt, "verdict_sha256": verdict} + for case, prompt, verdict in rows + } + + +def test_the_committed_seed_digest_is_the_pinned_one(): + """Runs anywhere: the committed digest hashes to the pin and spells each + of release 20261010's 677 judged cases once.""" + raw = driver.SEED_DIGEST.read_bytes() + assert hashlib.sha256(raw).hexdigest() == driver.SEED_DIGEST_SHA256 + seed = _committed_seed_digest() + assert driver.seed_digest_text(seed).encode() == raw + assert len(seed) == 677 + hexes = [value for item in seed.values() for value in item.values()] + assert all(len(h) == 64 and set(h) <= set("0123456789abcdef") for h in hexes) + + +def test_the_seed_is_release_20261006s_with_release_20261010s_rejudged_cases(): + """Runs anywhere: the committed digest is release 20261006's seed digest + (docs/haiku55/seed_digest.csv) with every case release 20261010 + re-judged (docs/haiku55/judge_provenance.json) written over it, and + nothing else: the rebuilt audit is the one that release published.""" + def read(path: Path) -> dict: + lines = path.read_text().splitlines() + return { + case: {"prompt_sha256": prompt, "verdict_sha256": verdict} + for case, prompt, verdict in (line.split(",") for line in lines[1:]) + } + + expected = read(REPO / "docs/haiku55/seed_digest.csv") + rejudged = json.loads((REPO / "docs/haiku55/judge_provenance.json").read_text()) + assert len(expected) == 674 and len(rejudged["verdicts"]) == 232 + for entry in rejudged["verdicts"]: + expected[entry["case_id"]] = { + "prompt_sha256": entry["prompt_sha256"], + "verdict_sha256": entry["verdict_sha256"], + } + assert _committed_seed_digest() == expected + + +def test_the_recovery_record_names_each_rejudged_case_once_at_its_hashes(): + """Runs anywhere: docs/musewonder/seed_recovery.json lists the 232 cases + rebuilt from the judges' transcripts, each at the hashes the Haiku 5.5 + provenance record committed, from one of its two logins.""" + record = json.loads((REPO / "docs/musewonder/seed_recovery.json").read_text()) + provenance = { + e["case_id"]: e + for e in json.loads((REPO / "docs/haiku55/judge_provenance.json").read_text())[ + "verdicts" + ] + } + cases = record["cases"] + assert sorted(c["case_id"] for c in cases) == sorted(provenance) + for item in cases: + entry = provenance[item["case_id"]] + assert item["prompt_sha256"] == entry["prompt_sha256"] + assert item["verdict_sha256"] == entry["verdict_sha256"] + assert record["counts"] == {"claude-18": 39, "pb-judge": 193} + groups = {"pb-judge": "isolated: setup-token 2", "claude-18": "isolated: subfleet lane claude-18"} + assert all(provenance[c["case_id"]]["group"] == groups[c["login"]] for c in cases) + + +def test_every_committed_adjudication_decides_a_case_the_seed_judged(): + """Runs anywhere: a recorded decision of release 20261010 rules on a judge + verdict, so its case is one of the seed's judged cases.""" + seed = _committed_seed_digest() + assert {driver.case_id(entry) for entry in driver.base_adjudications()} <= set(seed) + + +def test_the_committed_seed_digest_matches_the_real_seed(): + """Local only: the digest equals the rebuilt seed on this machine.""" + if not SEED.is_dir(): + pytest.skip("release 20261010's audit is not on this machine") + assert driver.seed_digest_text(driver.seed_digest(SEED)) == ( + driver.SEED_DIGEST.read_text() + ) + + +def test_every_rebuilt_verdict_is_its_transcripts_answer(): + """Local only: each rebuilt case's prompt and verdict are its kept + transcript's one user message and one accepted StructuredOutput answer, + as the runner's own checks read them (transcript_problems).""" + if not SEED.is_dir(): + pytest.skip("release 20261010's audit is not on this machine") + record = json.loads((REPO / "docs/musewonder/seed_recovery.json").read_text()) + for item in record["cases"]: + case = SEED / "cases" / item["case_id"] + meta = json.loads((case / "verdict.meta.json").read_text()) + assert meta["recovered"] and meta["judge_template_version"] == 1 + problems = driver.transcript_problems( + case / "claude.transcript.jsonl", + meta["judge_effort"], + (case / "prompt.md").read_bytes(), + json.loads((case / "verdict.json").read_text()), + ) + assert problems == [], (item["case_id"], problems) + + +def test_the_committed_adjudications_keep_the_seed_judge_verdicts(): + """Local only: every decision release 20261010 recorded keeps its seed + verdict's class.""" + if not SEED.is_dir(): + pytest.skip("release 20261010's audit is not on this machine") + from freeze_snapshot import verify_adjudications_keep_judge_verdicts + + verify_adjudications_keep_judge_verdicts( + driver.base_adjudications(), SEED / "cases" + ) + + +def test_each_20261010_entry_names_its_seed_verdict_as_its_sidecar_records_it(): + """Local only: the restate script requires a re-opened entry to name its + seed verdict, which it builds from the seed's sha256-bound sidecar: judge, + classes, flag, flag source and UTC day. It holds for all 70 entries.""" + if not SEED.is_dir(): + pytest.skip("release 20261010's audit is not on this machine") + from date_adds0928_judge_verdicts import WAVE_FLAGS, _bound_verdict + from restate_gpt61sol_adjudications import named_item, replaced_item + + waved = frozenset(json.loads(WAVE_FLAGS.read_text())) + base = driver.base_adjudications() + assert len(base) == 70 + for entry in base: + found = _bound_verdict(SEED / "cases" / driver.case_id(entry)) + assert found is not None, driver.case_id(entry) + key = f"{entry['scenario_id']}:{entry['variable']}" + assert named_item(entry) == replaced_item(found, key in waved), entry + + +@pytest.mark.slow +def test_every_seed_prompt_rerenders_on_v1_from_the_base(tmp_path): + """Local only: release 20261010's 47-model board, as BASE_COMMIT holds it, + renders exactly the seed's cases, and on judge template v1, the template + every seed verdict was judged on, every seed prompt byte for byte (this + reads the predictions from git into scratch). So check_prompt_changes may + attribute every changed or new prompt to Muse Wonder joining its case or + to the engine upgrade.""" + if not (SEED.is_dir() and GROUNDING.is_file()): + pytest.skip("release 20261010's audit or grounding is not on this machine") + from policybench.audit import prepare_audit + + bundle = tmp_path / "publish" / driver.RUN_NAME + (bundle / "us").mkdir(parents=True) + for name in (*driver.REFERENCE_FILES, "predictions.csv.gz"): + (bundle / "us" / name).write_bytes(base_blob(SNAPSHOT_PATH / name)) + (bundle / "annotations").mkdir() + for name in driver.ANNOTATION_FILES: + (bundle / "annotations" / name).write_bytes( + base_blob(Path("annotations") / driver.RUN_NAME / name) + ) + grounding = pd.read_csv(GROUNDING) + lookup = { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + audit = tmp_path / "audit" + arrow = hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string") + with ( + pd.option_context("future.infer_string", False) + if arrow + else contextlib.nullcontext() + ): + prepare_audit(bundle / "us", audit, grounding_lookup=lookup, template_version=1) + rendered = driver.seed_prompt_digests(audit) + seed = driver.seed_prompt_digests(SEED) + assert set(rendered) == set(seed) and len(rendered) == 677 + assert {case for case in rendered if rendered[case] != seed[case]} == set() + + +def test_the_stage_reopens_only_what_muse_wonder_or_the_upgrade_explains(): + """Local only, read-only: the live stage's prompt-changes.json is what + check_prompt_changes derives from its prompts and the bound seed, and + every re-opened case names Muse Wonder or is one the installed engine + upgrade changed.""" + if not (STAGE / driver.PROMPT_CHANGES).is_file(): + pytest.skip("needs the Muse Wonder stage") + recorded = json.loads((STAGE / driver.PROMPT_CHANGES).read_text()) + seed = json.loads((STAGE / "stage.json").read_text())["seed"] + upgraded = driver.upgraded_cases(STAGE) + derived = driver.check_prompt_changes( + STAGE / "audit", + {case: item["prompt_sha256"] for case, item in seed.items()}, + upgraded, + ) + assert {key: sorted(value) for key, value in derived.items()} == recorded + reopened = {*recorded["changed"], *recorded["added"]} + manifest = { + item["case_id"]: item + for item in map( + json.loads, (STAGE / "audit/cases.jsonl").read_text().splitlines() + ) + } + assert all( + NEW in manifest[case]["wrong_models"] or case in upgraded for case in reopened + ) + + +def test_the_stage_holds_the_installed_release_exclusions(): + """Local only, read-only: install-exclusions ran on the live stage and + bound the release's record in both copies.""" + receipt_path = STAGE / "stage.json" + if not receipt_path.is_file(): + pytest.skip("needs the Muse Wonder stage") + receipt = json.loads(receipt_path.read_text()) + installed = receipt.get("exclusions_installed") + if installed is None or receipt.get("references_installed"): + pytest.skip("install-exclusions has not run, or a build's record replaced it") + assert installed["sha256"] == release_sha() == driver.release_exclusions_sha256() + assert installed["base_sha256"] == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + assert installed["records"] == driver.RELEASE_EXCLUSIONS + assert receipt["files"][BOUND_EXCLUSIONS] == release_sha() + for target in (STAGE / "scoring", STAGE / "publish" / driver.RUN_NAME / "us"): + assert driver.digest(target / EXCLUSIONS_NAME) == release_sha() + + +# --- The release's exclusion record -------------------------------------------- + + +def test_the_spec_rules_no_exclusion(): + """d1252 and d1263 were open when the release was staged: the spec names + no proposal, no ruled adjudication and no restatement, so spec_records + finds none and the release record is release 20261010's.""" + spec = real_spec() + assert spec["proposals"] == [] and spec["adjudications"] == [] + assert spec["restated_adjudications"] == [] + assert driver.NEW_EXCLUSIONS == 0 + with mock.patch.object(driver, "ROOT", REPO): + assert driver.spec_records(spec) == [] + + +def test_the_release_exclusions_are_release_20261010s_unchanged(): + """With nothing ruled, build_release_exclusions returns release + 20261010's record, and its text is the committed file's bytes.""" + with mock.patch.object(driver, "ROOT", REPO): + record = driver.build_release_exclusions(base_record(), real_spec()) + assert record == base_record() + assert release_text().encode() == base_blob(SNAPSHOT_PATH / EXCLUSIONS_NAME) + assert release_sha() == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + assert len(record["exclusions"]) == driver.RELEASE_EXCLUSIONS == 58 + + +@pytest.mark.parametrize( + "extra", + [ + {"record_edits": [{"scenario_id": "scenario_000", "variable": "snap"}]}, + {"derivation_insert": {"before": " On", "text": " Extra."}}, + ], +) +def test_a_spec_that_edits_records_it_does_not_rule_is_refused(extra): + spec = {**real_spec(), **extra} + with mock.patch.object(driver, "ROOT", REPO): + with pytest.raises(SystemExit, match="rules no record"): + driver.build_release_exclusions(base_record(), spec) + + +# --- install-exclusions -------------------------------------------------------- + + +@pytest.fixture +def installable(tmp_path): + """A stage as prepare leaves it: release 20261010's record in the scoring + source and the bundle, bound in stage.json beside another input.""" + stage = tmp_path / "stage" + copies = [stage / "scoring", stage / "publish" / driver.RUN_NAME / "us"] + for target in copies: + target.mkdir(parents=True) + (target / EXCLUSIONS_NAME).write_bytes( + base_blob(SNAPSHOT_PATH / EXCLUSIONS_NAME) + ) + other = stage / "inputs" / SLUG / "predictions.csv" + other.parent.mkdir(parents=True) + other.write_text("model,scenario_id,variable\n") + receipt = { + "partial": False, + "early": False, + "files": { + BOUND_EXCLUSIONS: driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME], + f"inputs/{SLUG}/predictions.csv": driver.digest(other), + }, + "seed": {"us__s0__snap": {"prompt_sha256": "a" * 64}}, + } + (stage / "stage.json").write_text(json.dumps(receipt)) + return stage, copies, receipt + + +def test_install_exclusions_rebinds_the_stage_and_is_idempotent(installable): + stage, copies, receipt = installable + driver.install_exclusions(SimpleNamespace(stage_dir=stage)) + for target in copies: + assert (target / EXCLUSIONS_NAME).read_text() == release_text() + after = json.loads((stage / "stage.json").read_text()) + assert after["files"] == {**receipt["files"], BOUND_EXCLUSIONS: release_sha()} + assert after["exclusions_installed"] == { + "base_sha256": driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME], + "sha256": release_sha(), + "spec_sha256": driver.digest(REPO / driver.SPEC_PATH), + "records": 58, + } + assert { + k: v for k, v in after.items() if k not in ("files", "exclusions_installed") + } == {k: v for k, v in receipt.items() if k != "files"} + # Every binding the resume check reads holds after the rebind. + for name, pin in after["files"].items(): + assert driver.digest(stage / name) == pin + written = {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} + driver.install_exclusions(SimpleNamespace(stage_dir=stage)) + assert {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} == written + + +def test_install_exclusions_requires_the_bound_base_record_the_first_time( + installable, +): + stage, copies, receipt = installable + receipt["files"][BOUND_EXCLUSIONS] = "0" * 64 + (stage / "stage.json").write_text(json.dumps(receipt)) + before = {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} + with pytest.raises(SystemExit, match="is not release 20261010's"): + driver.install_exclusions(SimpleNamespace(stage_dir=stage)) + assert {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} == before + + +def test_install_exclusions_requires_a_bound_record(installable): + stage, _, receipt = installable + del receipt["files"][BOUND_EXCLUSIONS] + (stage / "stage.json").write_text(json.dumps(receipt)) + with pytest.raises(SystemExit, match="does not bind the exclusions"): + driver.install_exclusions(SimpleNamespace(stage_dir=stage)) + + +# --- Helpers the upgrade's adjudication tests share -------------------------- + + +def _all_verdicts(cases: Path, entries: list[dict]) -> Path: + """Verdicts that agree with each recorded decision's judge fields, as the + seed's do, for every case the base record decides.""" + for entry in entries: + case = cases / driver.case_id(entry) + if (case / "verdict.json").is_file(): + continue + case.mkdir(parents=True, exist_ok=True) + (case / "prompt.md").write_text(f"Classify {case.name}.\n") + verdict = { + **_verdict(["m1"], entry["judge_failure_source"]), + "case_failure_subtype": entry["judge_failure_subtype"], + "reference_suspect": bool(entry.get("judge_reference_suspect")), + } + write_verdict(case, verdict, judged_at_utc="2026-09-30T02:00:00+00:00") + return cases + + +def _write_stage_record(tmp_path: Path, record: dict) -> Path: + path = tmp_path / "publish" / driver.RUN_NAME / "annotations" / driver.ADJUDICATIONS + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(driver.record_text(record)) + return path + + +JSON_VALUES = st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=8), + lambda children: ( + st.lists(children, max_size=3) + | st.dictionaries(st.text(max_size=5), children, max_size=3) + ), + max_leaves=6, +) + + +# --- The date conventions ------------------------------------------------------ + +UPGRADE_WAVE = driver.DecisionWave("2026-10-13", excludes=True, affirms=False) +UPGRADE_WRITTEN = "2026-10-13" + + +def test_without_a_new_wave_the_date_conventions_are_release_20261010s(): + """The release rules nothing, so with no upgrade wave the record's date + conventions are release 20261010's, byte for byte.""" + conventions = base_adjudication_record()["date_conventions"] + assert driver.release_date_conventions(conventions) == conventions + + +def test_the_upgrade_wave_is_named_once_after_the_2026_10_10_wave(monkeypatch): + spec = {**real_spec(), "upgrade_adjudications_written_on": UPGRADE_WRITTEN} + monkeypatch.setattr(driver, "load_spec", lambda: copy.deepcopy(spec)) + conventions = base_adjudication_record()["date_conventions"] + out = driver.release_date_conventions(conventions, UPGRADE_WAVE) + day = UPGRADE_WAVE.day + named = driver.DATE_CONVENTIONS_BASE.replace( + " or 2026-10-10)", f", 2026-10-10 or {day})" + ) + sentence = ( + f" The {day} wave's decisions exclude the outputs the {day} engine " + "upgrade of the references moved onto an input the prompt does not " + f"state, and were written on {UPGRADE_WRITTEN} UTC." + ) + assert out == conventions.replace(driver.DATE_CONVENTIONS_BASE, named).replace( + driver.DATE_CONVENTIONS_ANCHOR, driver.DATE_CONVENTIONS_ANCHOR + sentence + ) + assert out.count(f"The {day} wave's decisions") == 1 + with pytest.raises(SystemExit, match="are not release 20261010's"): + driver.release_date_conventions(out, UPGRADE_WAVE) + + +def test_the_upgrade_wave_needs_its_written_day(): + spec = {k: v for k, v in real_spec().items() if k != "upgrade_adjudications_written_on"} + with mock.patch.object(driver, "load_spec", lambda: spec): + with pytest.raises(SystemExit, match="upgrade_adjudications_written_on"): + driver.release_date_conventions( + base_adjudication_record()["date_conventions"], UPGRADE_WAVE + ) + + +FILLER = st.text(alphabet="abc ().,0123456789-", max_size=20) + + +@settings(max_examples=150, deadline=None) +@given( + parts=st.lists(FILLER, min_size=5, max_size=5), + bases=st.integers(min_value=0, max_value=2), + anchors=st.integers(min_value=0, max_value=2), + wave=st.one_of( + st.none(), + st.builds( + driver.DecisionWave, + st.dates().map(lambda day: day.isoformat()), + st.booleans(), + st.just(True), + ), + ), +) +def test_release_date_conventions_requires_the_base_text_exactly_once( + parts, bases, anchors, wave +): + """For any text: it refuses unless the base wave list and the anchor each + occur exactly once; then it changes nothing without a wave, and with one + names it after 2026-10-10 and adds its sentence after the anchor.""" + a, b, c, d, e = parts + base, anchor = driver.DATE_CONVENTIONS_BASE, driver.DATE_CONVENTIONS_ANCHOR + text = a + (base + b) * bases + c + (anchor + d) * anchors + e + assume(text.count(base) == bases and text.count(anchor) == anchors) + spec = {**real_spec(), "upgrade_adjudications_written_on": UPGRADE_WRITTEN} + with mock.patch.object(driver, "load_spec", lambda: spec): + if (bases, anchors) != (1, 1): + with pytest.raises(SystemExit, match="are not release 20261010's"): + driver.release_date_conventions(text, wave) + return + out = driver.release_date_conventions(text, wave) + if wave is None: + assert out == text + return + named = base.replace(" or 2026-10-10)", f", 2026-10-10 or {wave.day})") + assert out.startswith(a + named + b + c + anchor + f" The {wave.day} wave's") + assert out.endswith(d + e) + + +# --- Triage -------------------------------------------------------------------- + +CONVENTIONS = ( + "Dates are UTC days. " + + driver.DATE_CONVENTIONS_BASE + + " The 2026-10-05 wave's decisions follow Max's rulings " + + driver.DATE_CONVENTIONS_ANCHOR + + " The rest is as before." +) + + +def _base_record(*entries) -> dict: + return {"date_conventions": CONVENTIONS, "adjudications": list(entries)} + + +def _record(*entries) -> str: + """A staged record: the date conventions name the wave, as the adjudicate + step writes them (load_spec must be patched with a written day).""" + return driver.record_text( + { + "date_conventions": driver.release_date_conventions(CONVENTIONS), + "adjudications": list(entries), + } + ) + + +@pytest.mark.parametrize( + "suspect, failure_source, report", + [ + (True, "llm_error", "reference-flags.csv"), + (False, "reference_engine_defect", "unresolved-rows.csv"), + (False, "prompt_ambiguity", "unresolved-rows.csv"), + ], +) +def test_triage_stops_and_records_flags_for_evidence_review( + tmp_path, monkeypatch, unruled_spec, suspect, failure_source, report +): + import policybench.audit + + stage = tmp_path / "stage" + bundle = stage / "publish" / driver.RUN_NAME + (bundle / "annotations").mkdir(parents=True) + rows = pd.DataFrame( + [ + { + "country": "us", + "scenario_id": "scenario_000", + "variable": "snap", + "model": NEW, + "failure_source": failure_source, + "failure_subtype": "thresholds_rates", + "annotation": "Evidence needed.", + "reference_suspect": suspect, + } + ] + ) + cases = pd.DataFrame( + [ + { + "country": "us", + "scenario_id": "scenario_000", + "variable": "snap", + "wrong_model_count": 1, + "reference_suspect": suspect, + "case_failure_source": failure_source, + "case_failure_subtype": "thresholds_rates", + "case_annotation": "Evidence needed.", + "reference_bug_hypothesis": "Check the threshold.", + } + ] + ) + monkeypatch.setattr(driver, "validate_verdicts", lambda *a, **kw: []) + monkeypatch.setattr(driver, "load_seed", lambda stage: {}) + monkeypatch.setattr(driver, "base_adjudication_record", lambda: _base_record()) + (stage / driver.PROMPT_CHANGES).write_text( + json.dumps({"added": [], "changed": [], "kept": []}) + ) + (bundle / "annotations" / driver.ADJUDICATIONS).write_text(_record()) + monkeypatch.setattr( + policybench.audit, + "collect_audit", + lambda *a, **kw: { + "missing": pd.DataFrame(), + "hedged": pd.DataFrame(), + "row": rows, + "case": cases, + }, + ) + with pytest.raises(SystemExit, match="triage required"): + driver.triage(SimpleNamespace(stage_dir=stage), bundle) + assert len(pd.read_csv(stage / report)) == 1 + assert not (stage / "release-ready.json").exists() + + +CASE = "us__scenario_000__snap" +DECISION = { + "country": "us", + "scenario_id": "scenario_000", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judged_on_utc": "2026-09-30", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "thresholds_rates", + "adjudicated_on": "2026-09-22", + "adjudicator": "developer", + "judge_reference_suspect": True, + "reference_verdict": "affirmed", + "reference_basis": "7 CFR 273.10", + "reasoning": "The judge counted one model's row. The allotment follows the rule.", +} + + +@pytest.fixture +def triage_stage(tmp_path, monkeypatch, unruled_spec): + """A one-case stage Muse Wonder re-opened, with a recorded decision.""" + import policybench.audit + + stage = tmp_path / "stage" + bundle = stage / "publish" / driver.RUN_NAME + (bundle / "annotations").mkdir(parents=True) + (stage / driver.PROMPT_CHANGES).write_text( + json.dumps({"added": [], "changed": [CASE], "kept": []}) + ) + case = stage / "audit/cases" / CASE + case.mkdir(parents=True) + write_verdict(case, {**_verdict([NEW]), "reference_suspect": True}) + record = bundle / "annotations" / driver.ADJUDICATIONS + record.write_text(_record(DECISION)) + monkeypatch.setattr(driver, "validate_verdicts", lambda *a, **kw: []) + monkeypatch.setattr(driver, "load_seed", lambda stage: {}) + monkeypatch.setattr( + driver, "base_adjudication_record", lambda: _base_record(DECISION) + ) + row = { + "country": "us", + "scenario_id": "scenario_000", + "variable": "snap", + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "reference_suspect": True, + } + rows = pd.DataFrame( + [ + {**row, "model": NEW, "annotation": "It used the 2025 threshold."}, + {**row, "model": "m1", "annotation": "It used the 2025 threshold."}, + ] + ) + cases = pd.DataFrame( + [ + { + **{k: v for k, v in row.items() if not k.startswith("failure")}, + "wrong_model_count": 2, + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_bug_hypothesis": "", + "case_annotation": "Both models used the 2025 threshold.", + } + ] + ) + monkeypatch.setattr( + policybench.audit, + "collect_audit", + lambda *a, **kw: { + "missing": pd.DataFrame(), + "hedged": pd.DataFrame(), + "row": rows.copy(), + "case": cases.copy(), + }, + ) + + def run(): + driver.triage(SimpleNamespace(stage_dir=stage), bundle) + annotations = bundle / "annotations" + return ( + json.loads(record.read_text())["adjudications"][0], + pd.read_csv(annotations / "us_case_notes.csv").iloc[0], + pd.read_csv(annotations / "us_audit_row_annotations.csv"), + ) + + return stage, record, run + + +SEED_ITEM = { + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "judge_reference_suspect": True, + "judged_on": "2026-09-30", +} +RESTATED = { + **DECISION, + "judged_on_utc": "2026-10-09", + "judge_rejudged_on": "2026-10-09", + "judge_previous": [SEED_ITEM], +} + + +def test_triage_accepts_its_own_record_unchanged(triage_stage): + stage, record, run = triage_stage + text = record.read_text() + entry, _, _ = run() + assert entry == DECISION and record.read_text() == text + + +def test_triage_lets_a_rejudged_case_restate_its_judge_fields(triage_stage): + stage, record, run = triage_stage + record.write_text(_record(RESTATED)) + entry, _, _ = run() + assert entry == RESTATED + + +@pytest.mark.parametrize( + "tamper", + [ + {"judge_model": "a-human-typed-this"}, + {"judge_previous": []}, + {"judge_rejudged_on": "1999-01-01"}, + # The appended item must be the verdict 20261010's entry names. + {"judge_previous": [{**SEED_ITEM, "judged_on": "2026-09-28"}]}, + {"judge_previous": [{**SEED_ITEM, "judge_failure_source": "reference_error"}]}, + {"judge_previous": [{**SEED_ITEM, "judge_failure_subtype": "other"}]}, + {"judge_previous": [{**SEED_ITEM, "judge_reference_suspect": False}]}, + ], +) +def test_triage_refuses_judge_fields_written_by_hand(triage_stage, tamper): + stage, record, run = triage_stage + record.write_text(_record({**RESTATED, **tamper})) + with pytest.raises(SystemExit, match="not restated by the restate script"): + run() + + +def test_triage_refuses_a_record_whose_bytes_hide_text(triage_stage): + stage, record, run = triage_stage + text = _record(DECISION).replace( + '"adjudicator": "developer"', + '"adjudicator": "SMUGGLED", "adjudicator": "developer"', + ) + record.write_text(text) + with pytest.raises(SystemExit, match="committed form"): + run() + + +@pytest.mark.parametrize( + "conventions", + [ + CONVENTIONS + " Edited.", + CONVENTIONS.replace( + driver.DATE_CONVENTIONS_LAST_WAVE, ", 2026-10-10 or 2026-10-13)" + ), + ], +) +def test_triage_refuses_date_conventions_other_than_the_waves( + triage_stage, conventions +): + """Without an upgrade wave, the staged record's date conventions are the + base's own text: neither an edit nor a wave the release does not add + passes.""" + stage, record, run = triage_stage + record.write_text( + driver.record_text( + {"date_conventions": conventions, "adjudications": [DECISION]} + ) + ) + with pytest.raises(SystemExit, match="beyond naming this release's waves"): + run() + + +@pytest.mark.parametrize( + "field, value", + [("adjudicated_failure_subtype", "other"), ("reasoning", "Rewritten.")], +) +def test_triage_refuses_any_other_change_to_a_recorded_decision( + triage_stage, field, value +): + stage, record, run = triage_stage + record.write_text(_record({**DECISION, field: value})) + with pytest.raises(SystemExit, match="change recorded decisions"): + run() + + +def test_triage_refuses_a_judge_rewrite_of_an_incumbent_only_case(triage_stage): + stage, record, run = triage_stage + (stage / driver.PROMPT_CHANGES).write_text( + json.dumps({"added": [], "changed": [], "kept": [CASE]}) + ) + record.write_text(_record(RESTATED)) + with pytest.raises(SystemExit, match="change recorded decisions"): + run() + + +def _amendments(stage, *items): + (stage / driver.AMENDMENTS).write_text(json.dumps({"amendments": list(items)})) + + +def _amend(field, old, new, **extra): + return { + "case_id": CASE, + "field": field, + "old": old, + "new": new, + "reason": "The re-judge's verdict no longer says this.", + **extra, + } + + +def test_triage_applies_exactly_the_listed_wording_amendments(triage_stage): + stage, record, run = triage_stage + _amendments( + stage, + _amend("reasoning", "one model's row", "two models' rows"), + _amend("case_annotation", "the 2025 threshold", "the 2025 threshold, held"), + _amend("annotation", "2025 threshold", "held 2025 threshold", model=NEW), + ) + entry, note, rows = run() + assert entry["reasoning"] == DECISION["reasoning"].replace( + "one model's row", "two models' rows" + ) + assert list(entry) == list(DECISION) + assert note.case_annotation.startswith("Both models used the 2025 threshold, held.") + assert "two models' rows" in note.case_annotation + annotations = dict(zip(rows.model, rows.annotation)) + assert annotations == { + NEW: "It used the held 2025 threshold.", + "m1": "It used the 2025 threshold.", + } + # Idempotent: a second triage applies nothing twice. + text = record.read_text() + assert run()[0] == entry and record.read_text() == text + + +@pytest.mark.parametrize( + "item", + [ + _amend("case_annotation", "no such text", "x"), + _amend("annotation", "2025 threshold", "x", model="not-a-model"), + # The adjudication sentence comes from the record; only a reasoning + # amendment may change it. + _amend("case_annotation", "The allotment follows the rule.", "It does."), + ], +) +def test_triage_refuses_an_amendment_it_cannot_apply_exactly(triage_stage, item): + from policybench.adjudications import AdjudicationError + + stage, _, run = triage_stage + _amendments(stage, item) + with pytest.raises( + (SystemExit, AdjudicationError), match="occurs|matches|adjudication sentence" + ): + run() + + +# --- Export, the scored count and the scope check ------------------------------ + +FABLE_USAGE = { + "costUsd": 54.109099999999955, + "costPerHousehold": 0.5410909999999995, + "totalTokens": 3850174, + "latencySeconds": 97.6780539804895, +} + + +def _stat(model, exact, n=driver.RELEASE_SCORED, **usage): + return { + "model": model, + "condition": "no_tools", + "score": exact / 100, + "exact": exact, + "n": n, + **usage, + } + + +def _incumbents(): + """47 rows as release 20261010 published them (1,926 scored outputs), + including Fable 5's batch usage and Ox Alpha's $0.""" + rows = [ + _stat(f"incumbent-{i:02d}", 50.0 + i / 7, n=driver.BASE_SCORED, costUsd=1.25) + for i in range(45) + ] + rows.append( + _stat("ox-alpha", 70.0, n=driver.BASE_SCORED, costUsd=0.0, costPerHousehold=0.0) + ) + rows.append( + _stat( + "claude-fable-5", + 60.0, + n=driver.BASE_SCORED, + costUsd=0.0, + costPerHousehold=0.0, + ) + ) + rows[-1].update(FABLE_USAGE) + return rows + + +def _exported(incumbents): + """What export_full_run returns on the release's record: every model on + 1,926 outputs and (MOCK) every incumbent's score moved, as an engine + upgrade's references would move it; Fable's usage is gone, and the + addition is new.""" + stats = copy.deepcopy(incumbents) + for row in stats: + row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) + row["score"] = row["exact"] / 100 + fable = stats[-1] + fable.update(costUsd=0.0, costPerHousehold=0.0) + del fable["totalTokens"], fable["latencySeconds"] + return stats + [_stat(NEW, 65.0, costUsd=0.13)] + + +def _scoped(incumbents): + """What release_20261006.export_payload returns with release 20261010's + record put back: every incumbent's published row, Fable's usage carried + (it does that itself), and the addition on 1,926 outputs.""" + return copy.deepcopy(incumbents) + [ + _stat(NEW, 65.5, n=driver.BASE_SCORED, costUsd=0.13) + ] + + +@pytest.fixture +def scope(monkeypatch): + """export_full_run, release_20261006.export_payload and the dashboard + schema replaced by fakes that hand back the given model stats and record + what they were called with.""" + import policybench.dashboard_schema + import policybench.full_run_export + + state = SimpleNamespace( + stats=None, scoped=None, calls=[], gates=[], references=[], scope_gates=[] + ) + + def export_full_run(run_dir, **kwargs): + state.calls.append(("export_full_run", Path(run_dir), kwargs)) + return {"countries": {"us": {"modelStats": copy.deepcopy(state.stats)}}} + + def export_payload( + bundle, base, exclusions=None, *, require_failure_annotations=True + ): + state.calls.append( + ("export_payload", Path(bundle), base, Path(exclusions).read_bytes()) + ) + state.scope_gates.append(require_failure_annotations) + # The reference files the scope export reads, where the bundle has them. + state.references.append( + { + name: (Path(bundle) / "us" / name).read_bytes() + for name in driver.REFERENCE_FILES + if (Path(bundle) / "us" / name).is_file() + } + ) + return {"countries": {"us": {"modelStats": copy.deepcopy(state.scoped)}}} + + def validate(payload, *, require_failure_annotations): + state.gates.append(require_failure_annotations) + return [] + + monkeypatch.setattr(policybench.full_run_export, "export_full_run", export_full_run) + monkeypatch.setattr(release_20261006, "export_payload", export_payload) + monkeypatch.setattr( + policybench.dashboard_schema, "validate_dashboard_payload", validate + ) + monkeypatch.setattr(driver, "base_exclusion_record", base_record) + + def build( + stats, + scoped, + live_stats, + *, + partial=False, + early=False, + bundle=None, + upgrade=None, + ): + state.stats, state.scoped = stats, scoped + live = {"countries": {"us": {"modelStats": live_stats}}} + return driver.build_payload( + bundle or Path("bundle"), + live, + partial=partial, + early=early, + upgrade=upgrade, + ) + + state.build = build + return state + + +def test_the_scope_check_allows_every_incumbent_change_the_base_record_explains(scope): + """Every incumbent's score moves on the release's record, and its + modelStats on release 20261010's record are the published ones: the + payload carries the release's rows, Fable 5's usage carried.""" + incumbents = _incumbents() + stats = _exported(incumbents) + payload = scope.build(stats, _scoped(incumbents), incumbents, bundle=Path("b")) + rows = payload["countries"]["us"]["modelStats"] + assert {row["n"] for row in rows} == {1926} + fable = next(row for row in rows if row["model"] == "claude-fable-5") + assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + assert fable["exact"] == incumbents[-1]["exact"] + 0.25 + (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls + assert run_dir == bundle == Path("b") + assert kwargs == {"countries": ["us"], "skip_app_data": True} + assert live == {"countries": {"us": {"modelStats": incumbents}}} + # The scope export scores on release 20261010's exclusion record, its + # pinned bytes. + assert ( + hashlib.sha256(record).hexdigest() + == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + ) + assert scope.gates == [True] + + +SCOPE_MUTATIONS = { + "value": lambda row: row.update(exact=row["exact"] + 1e-12), + "added_key": lambda row: row.update(extra=None), + "removed_key": lambda row: row.pop("score"), + "key_order": lambda row: row.update(model=row.pop("model")), + "int_to_float": lambda row: row.update(n=float(row["n"])), + "zero_cost_to_missing": lambda row: row.pop("costUsd"), + "cost_sign": lambda row: row.update(costUsd=-row["costUsd"]), + "renamed": lambda row: row.update(model=row["model"] + "-x"), +} + + +@settings( + max_examples=100, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + target=st.integers(min_value=0, max_value=45), + mutation=st.sampled_from(sorted(SCOPE_MUTATIONS)), +) +def test_the_scope_check_refuses_an_incumbent_change_the_ten_records_do_not_explain( + scope, target, mutation +): + """For every incumbent and every kind of change to its modelStats on + release 20261010's record, the build refuses and names it.""" + incumbents = _incumbents() + scoped = _scoped(incumbents) + SCOPE_MUTATIONS[mutation](scoped[target]) + with pytest.raises(SystemExit, match="incumbent modelStats drift") as refusal: + scope.build(_exported(incumbents), scoped, incumbents) + assert str(refusal.value).endswith(f"drift: {[incumbents[target]['model']]}") + + +def test_the_scope_check_refuses_a_scope_export_that_loses_a_model(scope): + incumbents = _incumbents() + with pytest.raises(SystemExit, match="scope export lost a model"): + scope.build(_exported(incumbents), _scoped(incumbents)[:-1], incumbents) + with pytest.raises(SystemExit, match=r"drift: \['incumbent-03'\]"): + scope.build( + _exported(incumbents), + [r for r in _scoped(incumbents) if r["model"] != "incumbent-03"] + + [_stat("stray", 1.0)], + incumbents, + ) + + +def test_every_model_is_scored_on_1926_outputs(scope): + incumbents = _incumbents() + for index in (0, 46, 47): + stats = _exported(incumbents) + stats[index]["n"] = 1920 + with pytest.raises(SystemExit, match="must be scored on 1926 outputs"): + scope.build(stats, _scoped(incumbents), incumbents) + assert [call[0] for call in scope.calls] == ["export_full_run"] * 3 + + +@settings( + max_examples=100, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + counts=st.lists( + st.sampled_from([1926, 1926, 1926, 1920, 1925, 1984, 0]), + min_size=48, + max_size=48, + ) +) +def test_the_scored_count_gate_passes_exactly_when_every_model_has_1926(scope, counts): + incumbents = _incumbents() + stats = _exported(incumbents) + for row, n in zip(stats, counts): + row["n"] = n + if set(counts) == {1926}: + scope.build(stats, _scoped(incumbents), incumbents) + else: + with pytest.raises(SystemExit, match="must be scored on 1926 outputs"): + scope.build(stats, _scoped(incumbents), incumbents) + + +def test_a_partial_build_skips_the_release_gates(scope): + stats = [_stat(f"incumbent-{i:02d}", 40.0, n=7) for i in range(47)] + payload = scope.build(stats + [_stat(NEW, 45.0, n=7)], None, [], partial=True) + assert "PARTIAL" in payload["stage2Status"] + assert [call[0] for call in scope.calls] == ["export_full_run"] + + +@settings(max_examples=60, deadline=None) +@given( + rows=st.lists( + st.fixed_dictionaries( + { + "exact": st.floats(allow_nan=False, allow_infinity=False), + "n": st.integers(min_value=0, max_value=1984), + } + ), + min_size=46, + max_size=46, + ), + target=st.integers(min_value=0, max_value=45), +) +def test_incumbent_drift_is_exactly_the_changed_rows(rows, target): + """Identical rows pass; a changed, dropped or reordered row is named.""" + previous = { + f"m{i:02d}": {"model": f"m{i:02d}", **row} for i, row in enumerate(rows) + } + stats = [copy.deepcopy(row) for row in previous.values()] + assert driver.incumbent_drift(stats, previous) == [] + name = f"m{target:02d}" + reordered = copy.deepcopy(stats) + reordered[target] = dict(reversed(list(reordered[target].items()))) + assert driver.incumbent_drift(reordered, previous) == [name] + dropped = [row for row in stats if row["model"] != name] + assert driver.incumbent_drift(dropped, previous) == [name] + + +@pytest.fixture +def exporting(tmp_path, monkeypatch, scope): + """A stage export can run on: release 20261010's references committed, + the release's exclusion record staged, Muse Wonder's pinned run and + an empty judge provenance record.""" + stage = tmp_path / "stage" + bundle = stage / "publish" / driver.RUN_NAME + monkeypatch.setattr( + driver, "SNAPSHOT", base_references(tmp_path / "committed-snapshot") + ) + base_references(bundle / "us") + (bundle / "us" / EXCLUSIONS_NAME).write_text(release_text()) + # No case is re-opened here, so the judge provenance record lists none. + record = tmp_path / "judge_provenance.json" + record.write_text(json.dumps(_provenance_record([]))) + monkeypatch.setattr(driver, "JUDGE_PROVENANCE", record) + monkeypatch.setattr(driver, "rejudged_cases", lambda stage: frozenset()) + pins = _write_run(stage, bundle) + monkeypatch.setattr(driver, "committed_input_pins", lambda: pins) + # Before the freeze: the pointer names release 20261010, whatever the + # working tree holds when the tests run. + monkeypatch.setattr(driver, "live_pointer", lambda: {"tag": driver.BASE_TAG}) + + def run(stats, live_stats, *, scoped=None, partial=False, early=False): + scope.stats = stats + scope.scoped = _scoped(live_stats) if scoped is None else scoped + args = SimpleNamespace(stage_dir=stage, partial=partial, early=early) + live = {"countries": {"us": {"modelStats": live_stats}}} + return driver.export(args, bundle, live) + + return stage, bundle, run, scope.gates + + +# Muse Wonder's synthetic run file, and an incumbent row beside its row. +HAIKU_RUN = ( + "model,scenario_id,variable,prediction,estimated_cost_usd,elapsed_seconds\n" + f"{NEW},scenario_000,snap,1200.0,0.0001,2.75\n" +) +PINNED = ("predictions.csv", "run_state.json") + + +def _write_run(stage, bundle) -> dict: + """Muse Wonder's run files as prepare stages them, the bundle's + predictions holding its row, and the prepare-time hashes in stage.json + and model-provenance.json; returns the run files' pins.""" + inputs = stage / "inputs" / SLUG + inputs.mkdir(parents=True, exist_ok=True) + (inputs / "predictions.csv").write_text(HAIKU_RUN) + fingerprint = {"model_id": NEW} + state = {"model": NEW, "treatment_fingerprint": fingerprint} + (inputs / "run_state.json").write_text(json.dumps(state)) + header, row = HAIKU_RUN.splitlines(keepends=True) + (bundle / "us/predictions.csv").write_text( + header + "incumbent,scenario_000,snap,900.0,0.002,3.5\n" + row + ) + names = [bundle / "us" / name for name in driver.REFERENCE_FILES] + names += [bundle / "us/predictions.csv", *(inputs / name for name in PINNED)] + files = {str(path.relative_to(stage)): driver.digest(path) for path in names} + (stage / "stage.json").write_text( + json.dumps({"partial": False, "early": False, "files": files}) + ) + provenance = { + "predictions_sha256": driver.digest(inputs / "predictions.csv"), + "treatment_fingerprint": fingerprint, + } + (stage / "model-provenance.json").write_text(json.dumps({NEW: provenance})) + return {SLUG: {name: driver.digest(inputs / name) for name in PINNED}} + + +def _evidence(stage, bundle): + paths = [bundle / "annotations/us_adjudications.json"] + paths += [stage / "audit/cases.jsonl", stage / "audit/schema.json"] + paths += [stage / driver.PROMPT_CHANGES, stage / driver.AMENDMENTS] + paths += [ + stage / "audit/cases/example" / name + for name in ( + "verdict.json", + "verdict.meta.json", + "prompt.md", + "claude.transcript.jsonl", + ) + ] + for path in paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"Evidence: {path.name}\n") + # The run files and prepare-time hashes the exporting fixture wrote. + paths += [bundle / "us/predictions.csv", stage / "stage.json"] + paths += [stage / "model-provenance.json"] + paths += [stage / "inputs" / SLUG / name for name in PINNED] + return paths + [bundle / "us" / name for name in driver.REFERENCE_FILES] + + +def test_strict_export_carries_fable_usage_and_binds_the_evidence(exporting): + stage, bundle, run, gate_calls = exporting + evidence = _evidence(stage, bundle) + incumbents = _incumbents() + stats = _exported(incumbents) + payload = run(stats, incumbents) + + output = stage / "data-board48.json" + assert ( + output.read_bytes() + == json.dumps({"countries": {"us": payload["countries"]["us"]}}).encode() + ) + assert (bundle / "data.json").read_bytes() == output.read_bytes() + fable = next( + s + for s in payload["countries"]["us"]["modelStats"] + if s["model"] == "claude-fable-5" + ) + assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + assert gate_calls == [True] + receipt = json.loads((stage / "release-ready.json").read_text()) + assert receipt["release_tag"] == driver.RELEASE_TAG + assert receipt["base_tag"] == driver.BASE_TAG + assert receipt["models"] == 48 and receipt["partial"] is False + assert receipt["base_sha256"] == driver.BASE_SHA256 + assert receipt["payload_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert receipt["files"] == { + str(path.relative_to(stage)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in evidence + } + assert receipt["files"][BOUND_EXCLUSIONS] == release_sha() + assert receipt["judge_provenance"] == { + "path": "docs/musewonder/judge_provenance.json", + "sha256": hashlib.sha256(driver.JUDGE_PROVENANCE.read_bytes()).hexdigest(), + } + + +def test_export_accepts_only_the_release_exclusion_record(exporting, monkeypatch): + """The release rules nothing, so the staged and committed records are both + release 20261010's; an edited copy of either is refused before a payload + or receipt is written.""" + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + incumbents = _incumbents() + staged = bundle / "us" / EXCLUSIONS_NAME + staged.write_text(release_text() + " ") + with pytest.raises(SystemExit, match="staged reference reference_exclusions.json"): + run(_exported(incumbents), incumbents) + staged.write_text(release_text()) + (driver.SNAPSHOT / EXCLUSIONS_NAME).write_text(release_text() + " ") + with pytest.raises( + SystemExit, match="committed reference reference_exclusions.json" + ): + run(_exported(incumbents), incumbents) + assert not (stage / "release-ready.json").exists() + assert not (stage / "data-board48.json").exists() + + +def test_a_re_export_after_the_freeze_accepts_the_frozen_exclusion_record( + exporting, monkeypatch +): + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + monkeypatch.setattr(driver, "live_pointer", lambda: {"tag": driver.RELEASE_TAG}) + (driver.SNAPSHOT / EXCLUSIONS_NAME).write_text(release_text()) + incumbents = _incumbents() + run(_exported(incumbents), incumbents) + assert (stage / "release-ready.json").is_file() + + +def test_export_writes_no_receipt_when_the_inputs_are_not_the_pinned_run( + exporting, monkeypatch +): + """Export checks Muse Wonder's staged run files against the committed + pins before it writes the payload or a receipt.""" + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + pins = {SLUG: {"predictions.csv": "0" * 64, "run_state.json": "0" * 64}} + monkeypatch.setattr(driver, "committed_input_pins", lambda: pins) + incumbents = _incumbents() + with pytest.raises(SystemExit, match=f"staged inputs/{SLUG}/.* is not the run"): + run(_exported(incumbents), incumbents) + assert not (stage / "release-ready.json").exists() + assert not (stage / "data-board48.json").exists() + + +def test_export_writes_nothing_when_the_scope_check_refuses(exporting): + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + incumbents = _incumbents() + scoped = _scoped(incumbents) + scoped[7]["exact"] += 0.5 + with pytest.raises(SystemExit, match=r"drift: \['incumbent-07'\]"): + run(_exported(incumbents), incumbents, scoped=scoped) + assert not (stage / "release-ready.json").exists() + assert not (stage / "data-board48.json").exists() + assert not (bundle / "data.json").exists() + + +def test_partial_export_never_gets_a_release_receipt(exporting): + stage, bundle, run, _ = exporting + stats = [_stat(f"incumbent-{i:02d}", 40.0) for i in range(47)] + run(stats + [_stat(NEW, 45.0)], [], partial=True, early=True) + payload = stage / "PARTIAL-data-board48.json" + assert "PARTIAL" in json.loads(payload.read_text())["stage2Status"] + assert (bundle / "data.json").read_bytes() == payload.read_bytes() + assert not (stage / "release-ready.json").exists() + + +def test_the_export_roster_must_be_the_incumbents_plus_the_addition(exporting): + _, _, run, _ = exporting + incumbents = _incumbents() + stats = _exported(incumbents) + stats[-1] = _stat("muse-wonder-impostor", 65.0) + with pytest.raises(SystemExit, match="roster"): + run(stats, incumbents) + with pytest.raises(SystemExit, match="47 incumbents"): + run(_exported(incumbents), incumbents[:-1] + [_stat(NEW, 1.0)]) + with pytest.raises(SystemExit, match="all 48 models"): + run(_exported(incumbents)[:-1], incumbents) + + +def test_export_refuses_a_staged_reference_off_its_pin(exporting): + _, bundle, run, _ = exporting + path = bundle / "us/reference_outputs.csv.meta.json" + path.write_bytes(path.read_bytes() + b"\n") + incumbents = _incumbents() + with pytest.raises(SystemExit, match="staged reference reference_outputs.csv.meta"): + run(_exported(incumbents), incumbents) + + +def test_export_refuses_a_committed_reference_off_its_pin( + exporting, pinned_copy, monkeypatch +): + _, _, run, _ = exporting + (pinned_copy / "scenarios.csv").write_text("scenario_id\n") + monkeypatch.setattr(driver, "SNAPSHOT", pinned_copy) + incumbents = _incumbents() + with pytest.raises(SystemExit, match="committed reference scenarios.csv"): + run(_exported(incumbents), incumbents) + + +# --- Judge provenance ---------------------------------------------------------- + +ISOLATED = "us__scenario_001__snap" +UNHARDENED = "us__scenario_002__snap" +# The prompt an isolated judge's transcript shows, and its case's prompt.md. +JUDGED = "Classify these wrong answers.\n" + + +def _event(kind, **fields): + return {"type": kind, **fields} + + +def _attachment(kind, **fields): + return _event("attachment", attachment={"type": kind, **fields}) + + +def _call(name, call_id, answer=None): + content = [{"type": "tool_use", "id": call_id, "name": name, "input": answer or {}}] + return _event("assistant", effort="xhigh", message={"content": content}) + + +def _result(call_id, error=False): + part = {"type": "tool_result", "tool_use_id": call_id, "content": "done"} + if error: + part["is_error"] = True + return _event("user", message={"content": [part]}) + + +def _clean_transcript(): + """An isolated judge's transcript, shaped like the GPT-6.1 Sol stage's real + ones: its one accepted StructuredOutput call answers the verdict the + provenance fixture writes.""" + return [ + _event("user", message={"content": JUDGED}), + _attachment("environment", snapshot={"isGitRepo": False}), + _attachment("model"), + _attachment("session_context", context={}), + _attachment("date"), + _attachment("prompt_snapshot"), + _event("assistant", effort="xhigh", message={"content": [{"type": "text"}]}), + _call("StructuredOutput", "call-1", _verdict([NEW])), + _attachment("structured_output"), + _result("call-1"), + ] + + +def _write_transcript(case, events): + (case / "claude.transcript.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events) + ) + + +# An isolated judge's sidecar fields: a token login on a lane whose account +# it declares, which the public record withholds. +ISOLATED_SIDECAR = { + "judge_effort": "xhigh", + "judge_isolation": {"tools": "none (--tools '')"}, + "judge_auth": {"method": "oauth_token", "account": None, "org": None}, + "judge_account_declared": "claude:lane@example.org (pb-judge)", +} +WITHHELD = "" + + +def _record_entry(case: Path, group: str) -> dict: + """The entry a true record gives a staged verdict: its hashes, its group + and isolation, and its sidecar's fields with addresses withheld.""" + meta = json.loads((case / "verdict.meta.json").read_text()) + return { + "case_id": case.name, + "group": group, + "isolated": "judge_isolation" in meta, + "judge_account_declared": ( + meta["judge_account_declared"].replace("lane@example.org", WITHHELD) + if "judge_account_declared" in meta + else None + ), + "judge_effort": meta.get("judge_effort"), + "judge_model_reported": meta["judge_model_reported"], + "judged_at_utc": meta["judged_at_utc"], + "judge_template_version": meta.get("judge_template_version"), + "verdict_sha256": driver.digest(case / "verdict.json"), + "prompt_sha256": driver.digest(case / "prompt.md"), + } + + +def _provenance_record(entries: list[dict], note: str = "How each was judged."): + counts: dict[str, int] = {} + for entry in entries: + counts[entry["group"]] = counts.get(entry["group"], 0) + 1 + return {"note": note, "counts": counts, "verdicts": entries} + + +GROUPS = {ISOLATED: "isolated: pb-judge", UNHARDENED: "unhardened: in-repo runner"} + + +@pytest.fixture +def provenance(tmp_path): + """Two new verdicts, one isolated and one not, and a true record of both.""" + cases = tmp_path / "audit/cases" + entries = [] + for name, isolated in ((ISOLATED, True), (UNHARDENED, False)): + case = cases / name + case.mkdir(parents=True) + (case / "prompt.md").write_text(JUDGED) + write_verdict(case, _verdict([NEW]), **(ISOLATED_SIDECAR if isolated else {})) + _write_transcript(case, _clean_transcript()) + entries.append(_record_entry(case, GROUPS[name])) + record = tmp_path / "judge_provenance.json" + + def verify(edit=lambda entries: None, edit_record=lambda record: None): + edited = copy.deepcopy(entries) + edit(edited) + written = _provenance_record(edited) + edit_record(written) + record.write_text(json.dumps(written)) + driver.verify_judge_provenance(cases, frozenset({ISOLATED, UNHARDENED}), record) + + return cases, verify + + +def test_the_judge_provenance_record_describes_the_staged_verdicts(provenance): + _, verify = provenance + verify() + + +def test_the_runners_attachment_allowlist_is_the_gates(): + script = (driver.ROOT / "scripts/run_audit_claude.sh").read_text() + line = next(line for line in script.splitlines() if line.startswith("ATTACHMENTS=")) + assert set(line.split('"')[1].split(",")) == driver.JUDGE_ATTACHMENTS + + +@pytest.mark.parametrize("case", [ISOLATED, UNHARDENED]) +def test_a_verdict_whose_isolation_disagrees_with_the_record_is_refused( + provenance, case +): + _, verify = provenance + + def flip(entries): + entry = next(e for e in entries if e["case_id"] == case) + entry["isolated"] = not entry["isolated"] + + with pytest.raises(SystemExit, match=f"{case}: the record says isolated"): + verify(flip) + + +def test_a_sidecar_that_drops_its_isolation_disagrees_with_the_record(provenance): + cases, verify = provenance + case = cases / ISOLATED + meta = json.loads((case / "verdict.meta.json").read_text()) + del meta["judge_isolation"] + (case / "verdict.meta.json").write_text(json.dumps(meta)) + with pytest.raises(SystemExit, match="sidecar lacks judge_isolation"): + verify() + + +@pytest.mark.parametrize( + "edit, problem", + [ + (lambda e: e.pop(), "does not list each re-opened case once"), + (lambda e: e.append(dict(e[0])), "does not list each re-opened case once"), + (lambda e: e[0].update(verdict_sha256="0" * 64), "names another verdict"), + (lambda e: e[1].update(prompt_sha256="0" * 64), "names another prompt"), + ], +) +def test_the_record_must_list_each_staged_new_verdict_once(provenance, edit, problem): + _, verify = provenance + with pytest.raises(SystemExit, match=problem): + verify(edit) + + +def _entry(case): + return lambda entries: next(e for e in entries if e["case_id"] == case) + + +# Each sidecar field the record copies, rewritten in one entry. +SIDECAR_FIELD_EDITS = { + "judge_effort": (ISOLATED, "max"), + "judge_model_reported": (UNHARDENED, ["claude-opus-5"]), + "judged_at_utc": (ISOLATED, "2026-10-09T08:00:00+00:00"), + "judge_account_declared": (ISOLATED, f"claude:{WITHHELD} (another lane)"), + "judge_account_declared_unhardened": (UNHARDENED, "a lane it never ran on"), + "judge_template_version": (ISOLATED, 1), +} + + +@pytest.mark.parametrize("edit", sorted(SIDECAR_FIELD_EDITS)) +def test_each_entrys_fields_must_be_its_sidecars(provenance, edit): + _, verify = provenance + case, value = SIDECAR_FIELD_EDITS[edit] + field = edit.removesuffix("_unhardened") + + def rewrite(entries): + _entry(case)(entries)[field] = value + + problem = f"{case}: its {field} .* is not the sidecar's" + with pytest.raises(SystemExit, match=problem): + verify(rewrite) + + +def test_the_record_withholds_the_sidecars_address_and_names_none(provenance): + """The entry carries the sidecar's declaration with its address withheld; + the address itself, there or anywhere else in the record, is refused.""" + _, verify = provenance + + def declared(entries): + _entry(ISOLATED)(entries)["judge_account_declared"] = ( + "claude:lane@example.org (pb-judge)" + ) + + with pytest.raises(SystemExit, match=r"is public and names .*e-mail address"): + verify(declared) + + def in_note(record): + record["note"] += " The lane is lane@example.org." + + with pytest.raises(SystemExit, match=r"is public and names .*record\.note"): + verify(edit_record=in_note) + + def in_group(entries): + for entry in entries: + entry["group"] = entry["group"].replace("pb-judge", "judge@example.org") + + with pytest.raises(SystemExit, match="is public and names"): + verify(in_group) + + +# The fixture record's true counts: one entry in each group. +GROUP_COUNTS = {group: 1 for group in GROUPS.values()} + + +@pytest.mark.parametrize( + "counts", [{}, {"isolated: pb-judge": 2}, {**GROUP_COUNTS, "extra": 0}] +) +def test_the_counts_must_be_the_tally_of_the_entries_groups(provenance, counts): + _, verify = provenance + with pytest.raises(SystemExit, match="counts .* are not the tally"): + verify(edit_record=lambda record: record.update(counts=counts)) + + +@pytest.mark.parametrize( + "auth", [None, {"method": "claude.ai", "account": "8f0c"}, {"method": None}] +) +def test_an_isolated_verdict_needs_a_token_login(provenance, auth): + cases, verify = provenance + meta_path = cases / ISOLATED / "verdict.meta.json" + meta = json.loads(meta_path.read_text()) + meta.pop("judge_auth") + if auth is not None: + meta["judge_auth"] = auth + meta_path.write_text(json.dumps(meta)) + problem = f"{ISOLATED}: the sidecar does not record a token login" + with pytest.raises(SystemExit, match=problem): + verify() + + +def test_the_record_and_its_entries_carry_only_checked_keys(provenance): + _, verify = provenance + + def claim(entries): + _entry(ISOLATED)(entries)["environment"] = "allowlisted" + + with pytest.raises(SystemExit, match=f"{ISOLATED}: the entry has keys"): + verify(claim) + with pytest.raises(SystemExit, match="has keys .*, not"): + verify(edit_record=lambda record: record.update(isolation="all allowlisted")) + + +def test_only_an_isolated_entrys_group_says_isolated(provenance): + _, verify = provenance + + def regroup(entries): + _entry(UNHARDENED)(entries)["group"] = "isolated: in-repo runner" + + with pytest.raises(SystemExit, match=f"{UNHARDENED}: its group .* disagrees"): + verify(regroup) + + +def _read_call(events): + events[-3:-3] = [_call("Read", "call-0"), _result("call-0")] + + +def _second_answer(events): + events += [_call("StructuredOutput", "call-2"), _result("call-2")] + + +def _no_answer(events): + del events[7:10] + + +def _user(content, **fields): + return _event("user", message={"content": content}, **fields) + + +def _forged_answer(events): + """A StructuredOutput call no assistant turn made, and its "result".""" + content = [{"type": "tool_use", "id": "call-9", "name": "StructuredOutput"}] + events += [ + _event("system", message={"content": content}), + _user([{"type": "tool_result", "tool_use_id": "call-9", "content": "A hint."}]), + ] + + +TRANSCRIPT_DEFECTS = { + "read_tool_call": (_read_call, r"tool calls \['Read'\]"), + "no_answer": (_no_answer, "0 accepted StructuredOutput"), + "two_answers": (_second_answer, "2 accepted StructuredOutput"), + "account_email": ( + lambda e: e[3].update(_attachment("session_context", context={"userEmail": 1})), + r"session context carrying \['userEmail'\]", + ), + "credential_org": ( + lambda e: e.append(_attachment("credential_org")), + "'credential_org' attachment", + ), + "skill_listing": ( + lambda e: e.append(_attachment("skill_listing")), + "'skill_listing' attachment", + ), + "git_repository": ( + lambda e: e[1].update(_attachment("environment", snapshot={"isGitRepo": True})), + "inside a git repository", + ), + "other_effort": (lambda e: e[6].update(effort="max"), "effort 'max'"), + "advisor": (lambda e: e[6].update(advisorModel="m"), "advisor model"), + "not_an_event": (lambda e: e.append("{"), "not an event"), + # The prompt binding: one user text message, prompt.md's, and otherwise + # only the results of the judge's own StructuredOutput calls. + "other_prompt": ( + lambda e: e[0].update(_user("Classify another case.\n")), + "its user message is not prompt.md", + ), + "no_prompt": (lambda e: e.pop(0), "0 user text messages"), + "second_text": ( + lambda e: e.insert(7, _user("Also weigh this hint.")), + "2 user text messages", + ), + "nudge_not_meta": ( + lambda e: e.insert(7, _user(driver.JUDGE_PROMPT_NUDGE)), + "2 user text messages", + ), + "meta_text_part": ( + lambda e: e.insert( + 7, _user([{"type": "text", "text": "A hint."}], isMeta=True) + ), + "not a StructuredOutput call's result", + ), + "stray_result": ( + lambda e: e.append(_user([{"type": "tool_result", "tool_use_id": "call-8"}])), + "not a StructuredOutput call's result", + ), + "forged_answer": (_forged_answer, "not a StructuredOutput call's result"), + # The verdict: the one accepted answer must be verdict.json. + "other_answer": ( + lambda e: e[7]["message"]["content"][0]["input"].update(rationale="Other."), + "its accepted StructuredOutput answer is not verdict.json", + ), + # Only the event types the isolated transcripts carry. + "unlisted_event": ( + lambda e: e.append(_event("progress", data={"phase": "thinking"})), + "an event of type 'progress'", + ), + # No account data outside the prompt's own text and the judge's words. + "email_in_system_prompt": ( + lambda e: e[5].update( + _attachment("prompt_snapshot", systemPrompt=["Mail max@example.org."]) + ), + r"an e-mail address at line 6\.attachment\.systemPrompt\[0\]", + ), + "account_key_in_environment": ( + lambda e: e[1].update( + _attachment( + "environment", + snapshot={"isGitRepo": False, "identity": {"accountUuid": "8f0c"}}, + ) + ), + r"key line 2\.attachment\.snapshot\.identity\.accountUuid", + ), + "email_beside_the_prompt": ( + lambda e: e[0].update(userEmail="max@example.org"), + r"key line 1\.userEmail", + ), + "account_key_beside_the_judges_words": ( + lambda e: e[6].update(organizationUuid="org-1"), + r"key line 7\.organizationUuid", + ), + "email_in_queue_operation": ( + lambda e: e.insert( + 0, _event("queue-operation", operation="enqueue", content="max@x.org") + ), + r"an e-mail address at line 1\.content", + ), + "git_status_key": ( + lambda e: e[4].update(_attachment("date", gitStatus="M a.py")), + r"key line 5\.attachment\.gitStatus", + ), +} + + +@pytest.mark.parametrize("defect", sorted(TRANSCRIPT_DEFECTS)) +def test_an_isolated_verdict_whose_transcript_fails_the_runners_checks_is_refused( + provenance, defect +): + cases, verify = provenance + edit, problem = TRANSCRIPT_DEFECTS[defect] + events = _clean_transcript() + edit(events) + _write_transcript(cases / ISOLATED, events) + with pytest.raises(SystemExit, match=f"{ISOLATED}: .*{problem}"): + verify() + + +def test_an_isolated_verdict_needs_its_transcript(provenance): + cases, verify = provenance + (cases / ISOLATED / "claude.transcript.jsonl").unlink() + with pytest.raises(SystemExit, match="no transcript"): + verify() + + +def test_an_answer_the_schema_refused_and_the_judge_gave_again_passes(provenance): + cases, verify = provenance + events = _clean_transcript() + events[7:7] = [_call("StructuredOutput", "call-0"), _result("call-0", error=True)] + _write_transcript(cases / ISOLATED, events) + verify() + + +def test_claude_codes_own_structured_output_nudge_passes(provenance): + """The judge answered in text, Claude Code nudged it (an isMeta user + message) and it called StructuredOutput.""" + cases, verify = provenance + events = _clean_transcript() + events.insert(7, _user(driver.JUDGE_PROMPT_NUDGE, isMeta=True)) + _write_transcript(cases / ISOLATED, events) + verify() + + +def test_an_address_in_the_prompt_or_the_judges_own_words_passes(provenance): + """The account-data rule reads neither the judged prompt's own text nor + the judge's turns.""" + cases, verify = provenance + case = cases / ISOLATED + prompt = "Classify these wrong answers; the filer wrote to irs@example.gov.\n" + (case / "prompt.md").write_text(prompt) + events = _clean_transcript() + events[0] = _user(prompt) + events[6]["message"]["content"] = [{"type": "text", "text": "irs@example.gov"}] + _write_transcript(case, events) + + def follow(entries): + entry = next(e for e in entries if e["case_id"] == ISOLATED) + entry["prompt_sha256"] = driver.digest(case / "prompt.md") + + verify(follow) + + +def test_the_runners_event_types_are_the_gates(): + script = (driver.ROOT / "scripts/run_audit_claude.sh").read_text() + line = next(line for line in script.splitlines() if line.startswith("EVENT_TYPES=")) + assert set(line.split('"')[1].split(",")) == driver.JUDGE_EVENT_TYPES + + +def test_the_runners_account_patterns_are_the_gates(): + script = (driver.ROOT / "scripts/run_audit_claude.sh").read_text() + assert driver.ACCOUNT_KEY.flags & re.IGNORECASE + assert f'account_key = re.compile(r"{driver.ACCOUNT_KEY.pattern}", re.I)' in script + assert f'email_address = re.compile(r"{driver.EMAIL_ADDRESS.pattern}")' in script + + +def test_the_runners_nudge_is_the_gates(): + script = (driver.ROOT / "scripts/run_audit_claude.sh").read_text() + line = next( + line for line in script.splitlines() if line.startswith("PROMPT_NUDGE=") + ) + assert line == f'PROMPT_NUDGE="{driver.JUDGE_PROMPT_NUDGE}"' + + +TRANSCRIPT_GATE = ( + "account_data", + "outside_the_judge", + "transcript_problems", + "withhold_addresses", +) + + +def _both_gates(path: Path, effort, prompt: bytes, verdict) -> tuple: + results = [] + for module in (driver, gpt61sol): + try: + results.append(module.transcript_problems(path, effort, prompt, verdict)) + except Exception as error: # noqa: BLE001 - compared, not swallowed + results.append((type(error).__name__, str(error))) + return tuple(results) + + +def test_this_drivers_transcript_gate_is_the_gpt61sol_drivers(): + """tests/test_run_audit_claude.py compares the runner with the GPT-6.1 Sol + driver's Python port of its transcript checks, so this driver's port must + be that one: the same source, the same constants and, on every defect the + tests name, the same problems.""" + for name in TRANSCRIPT_GATE: + assert inspect.getsource(getattr(driver, name)) == inspect.getsource( + getattr(gpt61sol, name) + ), name + for name in ( + "JUDGE_ATTACHMENTS", + "JUDGE_EVENT_TYPES", + "JUDGE_PROMPT_NUDGE", + "JUDGE_PROVENANCE_KEYS", + "WITHHELD_ADDRESS", + "JUDGE_MODEL", + ): + assert getattr(driver, name) == getattr(gpt61sol, name), name + # This release's record also states each new verdict's judge template. + assert driver.JUDGE_SIDECAR_FIELDS == ( + *gpt61sol.JUDGE_SIDECAR_FIELDS, + "judge_template_version", + ) + assert driver.JUDGE_PROVENANCE_ENTRY_KEYS == ( + gpt61sol.JUDGE_PROVENANCE_ENTRY_KEYS | {"judge_template_version"} + ) + for name in ("ACCOUNT_KEY", "EMAIL_ADDRESS"): + ours, theirs = getattr(driver, name), getattr(gpt61sol, name) + assert (ours.pattern, ours.flags) == (theirs.pattern, theirs.flags), name + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "claude.transcript.jsonl" + for defect in [None, *sorted(TRANSCRIPT_DEFECTS)]: + events = _clean_transcript() + if defect is not None: + TRANSCRIPT_DEFECTS[defect][0](events) + _write_transcript(path.parent, events) + ours, theirs = _both_gates(path, "xhigh", JUDGED.encode(), _verdict([NEW])) + assert ours == theirs, defect + # The comparison is not vacuous: each named defect is a problem. + assert (ours == []) == (defect is None), defect + + +JSON_EVENT = st.fixed_dictionaries( + {"type": st.sampled_from(sorted(driver.JUDGE_EVENT_TYPES | {"system", "other"}))}, + optional={ + "message": st.fixed_dictionaries( + {}, + optional={ + "content": st.text(max_size=12) + | st.lists( + st.fixed_dictionaries( + { + "type": st.sampled_from( + ["text", "tool_use", "tool_result", "server_tool_use"] + ) + }, + optional={ + "id": st.sampled_from(["c1", "c2"]), + "tool_use_id": st.sampled_from(["c1", "c2"]), + "name": st.sampled_from(["StructuredOutput", "Read"]), + "input": JSON_VALUES, + "is_error": st.booleans(), + }, + ), + max_size=3, + ) + }, + ), + "attachment": st.fixed_dictionaries( + {"type": st.sampled_from(sorted(driver.JUDGE_ATTACHMENTS | {"skill"}))}, + optional={"context": JSON_VALUES, "snapshot": JSON_VALUES}, + ), + "effort": st.sampled_from(["xhigh", "max"]), + "isMeta": st.booleans(), + "userEmail": st.just("a@b.org"), + }, +) + + +@settings(max_examples=200, deadline=None) +@given( + events=st.lists(JSON_EVENT | JSON_VALUES, max_size=6), + defects=st.lists(st.sampled_from(sorted(TRANSCRIPT_DEFECTS)), max_size=3), + effort=st.none() | st.just("xhigh"), +) +def test_the_transcript_gates_agree_on_any_transcript(events, defects, effort): + """Differential: on arbitrary event lists, and on any combination of the + named defects, both drivers' transcript checks give the same answer.""" + clean = _clean_transcript() + for defect in defects: + try: + TRANSCRIPT_DEFECTS[defect][0](clean) + except (IndexError, KeyError, TypeError, AttributeError): + pass + with tempfile.TemporaryDirectory() as scratch: + for lines in (events, clean, clean + events): + path = Path(scratch) / "claude.transcript.jsonl" + path.write_text("".join(json.dumps(line) + "\n" for line in lines)) + ours, theirs = _both_gates(path, effort, JUDGED.encode(), _verdict([NEW])) + assert ours == theirs + + +def test_a_prompt_changed_after_its_judge_ran_is_refused(provenance): + """A prompt.md replaced after judging, with the record following it, still + disagrees with the prompt the judge's transcript shows.""" + cases, verify = provenance + (cases / ISOLATED / "prompt.md").write_text("Classify these, and one more.\n") + + def follow(entries): + entry = next(e for e in entries if e["case_id"] == ISOLATED) + entry["prompt_sha256"] = driver.digest(cases / ISOLATED / "prompt.md") + + with pytest.raises(SystemExit, match=f"{ISOLATED}: its user message is not"): + verify(follow) + + +def test_an_unhardened_verdicts_transcript_is_not_held_to_isolation(provenance): + cases, verify = provenance + events = _clean_transcript() + _read_call(events) + _write_transcript(cases / UNHARDENED, events) + verify() + + +def test_export_writes_no_receipt_when_the_record_disagrees(exporting, monkeypatch): + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + case = stage / "audit/cases" / ISOLATED + case.mkdir(parents=True) + (case / "prompt.md").write_text(JUDGED) + write_verdict(case, _verdict([NEW]), **ISOLATED_SIDECAR) + _write_transcript(case, _clean_transcript()) + entry = _record_entry(case, GROUPS[ISOLATED]) + driver.JUDGE_PROVENANCE.write_text(json.dumps(_provenance_record([entry]))) + monkeypatch.setattr(driver, "rejudged_cases", lambda stage: frozenset({ISOLATED})) + incumbents = _incumbents() + run(_exported(incumbents), incumbents) + receipt = json.loads((stage / "release-ready.json").read_text()) + assert f"audit/cases/{ISOLATED}/claude.transcript.jsonl" in receipt["files"] + + (stage / "release-ready.json").unlink() + (stage / "data-board48.json").unlink() + entry["isolated"] = False + driver.JUDGE_PROVENANCE.write_text(json.dumps(_provenance_record([entry]))) + with pytest.raises(SystemExit, match="disagree with docs/musewonder"): + run(_exported(incumbents), incumbents) + assert not (stage / "release-ready.json").exists() + assert not (stage / "data-board48.json").exists() + + +def test_the_committed_provenance_record_is_public_and_tallied(): + """Once committed: the record is json.dumps(indent=1), ASCII-escaped, with + a trailing newline; it names no e-mail address, and its counts are the + tally of its entries' groups.""" + if not driver.JUDGE_PROVENANCE.is_file(): + pytest.skip("docs/musewonder/judge_provenance.json is not written yet") + text = driver.JUDGE_PROVENANCE.read_text() + record = json.loads(text) + assert text == json.dumps(record, indent=1) + "\n" and text.isascii() + assert not driver.EMAIL_ADDRESS.search(text) + groups = [entry["group"] for entry in record["verdicts"]] + assert record["counts"] == {group: groups.count(group) for group in groups} + + +def test_the_committed_provenance_record_describes_the_stage(): + """Local only, read-only, once written: the record passes the gate + against the live stage, every isolated transcript included.""" + if not driver.JUDGE_PROVENANCE.is_file(): + pytest.skip("docs/musewonder/judge_provenance.json is not written yet") + if not (WORKING_STAGE / "audit/cases").is_dir(): + pytest.skip("needs the Muse Wonder working stage") + driver.verify_judge_provenance( + WORKING_STAGE / "audit/cases", + driver.rejudged_cases(WORKING_STAGE), + driver.JUDGE_PROVENANCE, + ) + + +# --- The fold ------------------------------------------------------------------ + +REFERENCE = pd.read_csv(driver.SNAPSHOT / "reference_outputs.csv")[driver.KEY] +TEXT = st.text( + alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\x00"), + max_size=12, +) +NUMBERS = st.one_of( + st.none(), + st.floats(min_value=-1e7, max_value=1e7, allow_nan=False).map( + lambda x: round(x, 2) + ), + st.integers(min_value=0, max_value=10**6).map(float), +) + + +def _frame(model: str, rng, pools: dict) -> pd.DataFrame: + count = len(REFERENCE) + frame = REFERENCE.copy() + frame.insert(0, "model", model) + for column, pool in pools.items(): + frame[column] = [pool[i] for i in rng.integers(0, len(pool), count)] + frame["provider_resolved_model"] = model + return frame + + +@settings(max_examples=12, deadline=None) +@given( + incumbents=st.integers(min_value=1, max_value=3), + seed=st.integers(min_value=0, max_value=2**32 - 1), + numbers=st.lists(NUMBERS, min_size=1, max_size=6), + words=st.lists(TEXT, min_size=1, max_size=6), + extra_column=st.booleans(), +) +def test_the_fold_keeps_incumbent_rows_and_adds_1984_rows_for_the_addition( + tmp_path_factory, incumbents, seed, numbers, words, extra_column +): + """Any board: incumbent rows survive byte for byte; the addition is exact. + + Prediction columns keep their production types: numeric columns are + floats, and text columns carry prose (a letter before any digits). + """ + import freeze_snapshot as freezer + import numpy as np + + from policybench.fold_board import fold_board + + rng = np.random.default_rng(seed) + prose = [f"x{word}" for word in words] + [None] + pools = { + "prediction": numbers, + "explanation": prose, + "raw_response": prose, + "error": [None, "Timeout, retried", 'said "no"'], + "prompt_tokens": numbers, + "completion_tokens": numbers, + } + root = tmp_path_factory.mktemp("fold") + base = pd.concat( + [_frame(f"incumbent-{i}", rng, pools) for i in range(incumbents)], + ignore_index=True, + ) + base_path = root / "base-predictions.csv" + base.to_csv(base_path, index=False) + addition = _frame(NEW, rng, pools) + if extra_column: + addition["provider_system_fingerprint"] = "fp_1" + addition = addition.drop(columns=["raw_response"]) + addition_path = root / "fold.csv" + addition.to_csv(addition_path, index=False) + scoring = root / "scoring" + scoring.mkdir() + for name in driver.REFERENCE_FILES: + shutil.copyfile(driver.SNAPSHOT / name, scoring / name) + + result = fold_board( + base_path, [addition_path], scoring, root / "fold", export=False + ) + + assert result["excluded"] == {} and result["models"] == incumbents + 1 + folded_path = root / "fold/us/predictions.csv" + folded = pd.read_csv(folded_path, low_memory=False) + new_rows = folded[folded.model == NEW] + assert len(new_rows) == len(REFERENCE) == 1984 + assert set(new_rows[driver.KEY].itertuples(index=False, name=None)) == set( + REFERENCE.itertuples(index=False, name=None) + ) + reread = pd.read_csv(base_path, low_memory=False) + text = {"dtype": str, "keep_default_na": False} + base_text = pd.read_csv(base_path, **text) + folded_text = pd.read_csv(folded_path, **text)[base_text.columns] + for i in range(incumbents): + model = f"incumbent-{i}" + assert freezer._model_prediction_rows_sha256( + folded, model + ) == freezer._model_prediction_rows_sha256(reread, model) + before = base_text[base_text.model == model].reset_index(drop=True) + after = folded_text[folded_text.model == model].reset_index(drop=True) + pd.testing.assert_frame_equal(before, after) + for name in driver.REFERENCE_FILES: + assert (root / "fold/us" / name).read_bytes() == ( + driver.SNAPSHOT / name + ).read_bytes() + + +# --- One tag constant ---------------------------------------------------------- + + +def test_the_release_tag_is_named_in_one_place(): + """RELEASE_TAG is the only spelling of the new tag in the driver's scripts, + tests and design note; the spec, which the freeze checks against it, is + the one input that names it too.""" + source = (REPO / "scripts/finish_musewonder.py").read_text().splitlines() + assert sum(driver.RELEASE_TAG in line for line in source) == 1 + for relative in ( + "scripts/freeze_musewonder.py", + "docs/musewonder/design.md", + "tests/test_finish_musewonder.py", + ): + path = REPO / relative + if path.is_file(): + assert driver.RELEASE_TAG not in path.read_text(), relative + assert real_spec()["release_tag"] == driver.RELEASE_TAG + + +def test_the_freeze_tests_do_not_spell_the_release_tag(): + """The same rule for tests/test_freeze_musewonder.py, kept apart so a + failure names that file's lines.""" + path = REPO / "tests/test_freeze_musewonder.py" + if not path.is_file(): + pytest.skip("tests/test_freeze_musewonder.py does not exist yet") + lines = [ + number + for number, line in enumerate(path.read_text().splitlines(), 1) + if driver.RELEASE_TAG in line + ] + assert lines == [], f"tests/test_freeze_musewonder.py spells the tag on {lines}" + + +def _provenance_case(root: Path, case_id: str, meta: dict) -> None: + case = root / case_id + case.mkdir(parents=True) + (case / "prompt.md").write_text("prompt") + (case / "verdict.json").write_text("{}") + (case / "verdict.meta.json").write_text(json.dumps(meta)) + + +def test_the_provenance_record_groups_each_verdict_by_its_declared_account(tmp_path): + """Each entry copies its sidecar's fields (an address withheld), hashes its + verdict and prompt, and takes its group from the declared account; the + counts are the tally of the groups.""" + lane = "claude:max@axiom.org (subfleet lane claude-18)" + token = next(k for k in driver.PROVENANCE_GROUPS if "setup-token" in k) + for case_id, declared in (("us__a__x", token), ("us__b__y", lane)): + _provenance_case( + tmp_path, + case_id, + { + "judge_account_declared": declared, + "judge_effort": "xhigh", + "judge_model_reported": ["claude-opus-5-5"], + "judged_at_utc": "2026-10-09T11:00:00+00:00", + "judge_isolation": {"cwd": "empty"}, + }, + ) + record = driver.judge_provenance_record( + tmp_path, frozenset({"us__a__x", "us__b__y"}) + ) + assert set(record) == driver.JUDGE_PROVENANCE_KEYS + assert record["counts"] == { + "isolated: setup-token 2": 1, + "isolated: subfleet lane claude-18": 1, + } + by_case = {entry["case_id"]: entry for entry in record["verdicts"]} + assert set(by_case["us__a__x"]) == driver.JUDGE_PROVENANCE_ENTRY_KEYS + assert by_case["us__b__y"]["judge_account_declared"] == ( + f"claude:{driver.WITHHELD_ADDRESS} (subfleet lane claude-18)" + ) + assert by_case["us__a__x"]["prompt_sha256"] == hashlib.sha256(b"prompt").hexdigest() + assert not driver.account_data(record, "record", keys=False) + + +@pytest.mark.parametrize( + "meta", + [ + {"judge_account_declared": "someone else", "judge_isolation": {}}, + {"judge_account_declared": ("claude:max@axiom.org (subfleet lane claude-18)")}, + ], +) +def test_the_provenance_record_refuses_an_unknown_account_or_an_unisolated_verdict( + tmp_path, meta +): + _provenance_case(tmp_path, "us__a__x", meta) + with pytest.raises(SystemExit): + driver.judge_provenance_record(tmp_path, frozenset({"us__a__x"})) + + +def test_the_provenance_record_of_the_live_stage_passes_the_export_gate(): + """Local: once every re-opened case has its verdict, the record built from + the working stage's sidecars passes verify_judge_provenance (transcripts + included), so export accepts it.""" + stage = WORKING_STAGE + if not (stage / "prompt-changes.json").is_file(): + pytest.skip("no live stage") + cases = stage / "audit" / "cases" + rejudged = driver.rejudged_cases(stage) + if not all((cases / case / "verdict.json").is_file() for case in rejudged): + pytest.skip("the live stage is still being judged") + record = driver.judge_provenance_record(cases, rejudged) + with tempfile.TemporaryDirectory() as scratch: + path = Path(scratch) / "judge_provenance.json" + path.write_text(json.dumps(record)) + driver.verify_judge_provenance(cases, rejudged, path) + + +# --- An engine upgrade of the references (--step install-references) --------- +# +# MOCK DATA. Every build below is synthetic: its engine version, its moves, its +# new exclusion records, its traces and its narratives are invented for these +# tests. They are shaped like the hub's preview sweep (reviews/policybench-pe- +# upgrade-2026-10-09/cells_pe2.37.1.md) and like the records reference_audit/ +# 2026-10-09-engine-upgrade/scripts/build_references_upgrade.py writes, but +# they are not published references and must never be read as results. + +MOCK_ENGINE = "policyengine-us 2.40.9" +MOCK_DATE = "2026-10-12" +CSV_NAME, META_NAME = "reference_outputs.csv", "reference_outputs.csv.meta.json" +TRACES_NAME = "reference_traces.json" +EXPLANATIONS_PATH = Path("annotations") / driver.RUN_NAME / driver.EXPLANATIONS_NAME +STATE_TAX = "state_income_tax_before_refundable_credits" +TAIL = ("scenario_023", "head_medicaid_eligible") +# A scored output the mock engine moves beyond the tolerance, approved. +MOCK_APPROVED = {("scenario_076", STATE_TAX): 6838.34} +# Two scored outputs the mock engine moves onto an unstated county: new +# unlisted-input exclusions, frozen at the mock engine's values. +MOCK_NEW = { + ("scenario_001", "local_income_tax"): 670.55, + ("scenario_002", "local_income_tax"): 1348.83, +} +# Four release 20261010 engine-defect records the mock engine fixes, +# regenerated near their records' alternative values (record targets). +MOCK_REGENERATED = { + ("scenario_007", STATE_TAX): 651.32, + ("scenario_053", STATE_TAX): 2176.06, + ("scenario_100", "federal_refundable_credits"): 822.29, + ("scenario_119", "federal_refundable_credits"): 3366.07, +} +MOCK_REGENERATED_CASES = frozenset(driver.output_case(k) for k in MOCK_REGENERATED) +# A release 20261010 engine-defect record (r06 + r32, alternative $0) the mock +# engine fixes. +WI_042 = ("scenario_042", STATE_TAX) +WI_042_CASE = "us__scenario_042__state_income_tax_before_refundable_credits" +# A release 20261010 unlisted-input record (no engine can fix it). +MEDICARE_007 = ("scenario_007", "head_medicare_eligible") + + +def sha(data) -> str: + raw = data.read_bytes() if isinstance(data, Path) else data + return hashlib.sha256(raw).hexdigest() + + +def mock_new_record(key, value, engine=MOCK_ENGINE) -> dict: + """MOCK: a new unlisted-input exclusion on the mock engine.""" + return { + "scenario_id": key[0], + "variable": key[1], + "reason_code": "reference_depends_on_unlisted_input", + "unlisted_input": "county_fips", + "alternative_reading": "MOCK: the household lives in a county with no tax.", + "frozen_value": value, + "alternative_value": 0.0, + "engine_version": engine, + "decided_on": MOCK_DATE, + "decided_by": "developer", + "note": "MOCK record for the install-references tests.", + } + + +def csv_with(base: bytes, values: dict) -> bytes: + """The base reference CSV with each listed output's value field rewritten + and every other row byte for byte, as the builder writes it.""" + lines = base.decode().splitlines(keepends=True) + out = [lines[0]] + for line in lines[1:]: + fields = next(csv.reader([line])) + if (fields[0], fields[1]) in values: + fields[2] = repr(float(values[(fields[0], fields[1])])) + buffer = io.StringIO() + csv.writer(buffer, lineterminator="\n").writerow(fields) + line = buffer.getvalue() + out.append(line) + return "".join(out).encode() + + +def explanations_with(base: bytes, values: dict) -> bytes: + """The base explanations with each listed output's row rewritten as the + narratives script rewrites it (MOCK narrative text).""" + rows = list(csv.reader(io.StringIO(base.decode(), newline=""))) + header = rows[0] + index = {name: header.index(name) for name in header} + for row in rows[1:]: + key = (row[index["scenario_id"]], row[index["variable"]]) + if key in values: + row[index["reference_value"]] = repr(float(values[key])) + row[index["trace_lines"]] = "3" + row[index["explanation"]] = ( + f"MOCK narrative: PolicyEngine calculated ${values[key]:,.2f}." + ) + row[index["error"]] = "" + buffer = io.StringIO() + csv.writer(buffer, lineterminator="\n").writerows(rows) + return buffer.getvalue().encode() + + +def _asked_target(target: dict) -> dict: + """The action's form of an expanded target, as build_references_upgrade.py + reads it: a record target is its kind; a fix_modules target names its + modules and evidence file.""" + if target["kind"] != "fix_modules": + return {"kind": target["kind"]} + return { + "kind": "fix_modules", + "modules": [m["module"] for m in target["modules"]], + "evidence": {"path": target["evidence"], "sha256": target["evidence_sha256"]}, + } + + +def write_build( + root: Path, + base_files: dict, + base_explanations: bytes, + release_doc: dict, + ruled_keys, + *, + approved=None, + regenerated=None, + added=None, + rechecked=None, + engine=MOCK_ENGINE, + targets=None, +) -> SimpleNamespace: + """MOCK: a reference build in the shape build_references_upgrade.py writes. + + ``approved`` and ``regenerated`` map outputs to their new values, + ``added`` maps outputs to new exclusion records (frozen at the new value), + ``rechecked`` maps kept excluded outputs to their mock-engine values. The + build's exclusion record is ``release_doc`` less the regenerated records, + with the added ones before the trailing audit record. Each regeneration's + audited target is ``targets[key]``, or by default its record's + alternative_value (a "record" target), as the builder states it. + """ + approved, regenerated = dict(approved or {}), dict(regenerated or {}) + targets = dict(targets or {}) + added, rechecked = dict(added or {}), dict(rechecked or {}) + base_values = driver.reference_values(base_files[CSV_NAME]) + records = {driver.spec_key(r): r for r in release_doc["exclusions"]} + moves = {**approved, **regenerated} + moves |= {key: float(record["frozen_value"]) for key, record in added.items()} + moved = {k: v for k, v in moves.items() if float(v) != base_values[k]} + csv_blob = csv_with(base_files[CSV_NAME], moved) + doc = copy.deepcopy(release_doc) + kept = [r for r in doc["exclusions"] if driver.spec_key(r) not in regenerated] + tail = kept[-1:] if kept and driver.spec_key(kept[-1]) == TAIL else [] + doc["exclusions"] = kept[: len(kept) - len(tail)] + doc["exclusions"] += [added[k] for k in sorted(added)] + tail + doc["derivation"] += f" MOCK: on {MOCK_DATE} the references moved to {engine}." + number = engine.removeprefix("policyengine-us ") + kept_ruled = [ + {"scenario_id": k[0], "variable": k[1]} + for k in sorted(set(ruled_keys) - set(regenerated)) + ] + actions = { + "engine": engine, + "previous_engine": driver.BASE_ENGINE, + "date": MOCK_DATE, + "draft": False, + "approved": [ + { + "scenario_id": k[0], + "variable": k[1], + "value": v, + "cause": "mock_move", + "basis": "MOCK basis.", + } + for k, v in sorted(approved.items()) + ], + "regenerated_exclusions": [ + { + "scenario_id": k[0], + "variable": k[1], + "alternative_value": records[k]["alternative_value"], + "tolerance": 1.0, + "upstream": "MOCK upstream fix", + "basis": "MOCK basis.", + **({"target": _asked_target(targets[k])} if k in targets else {}), + } + for k in sorted(regenerated) + ], + "new_exclusions": [added[k] for k in sorted(added)], + "excluded_rechecked": [ + {"scenario_id": k[0], "variable": k[1], "reason": "MOCK reason."} + for k in sorted(rechecked) + ], + "kept_exclusions_from_release": kept_ruled, + } + actions_blob = (json.dumps(actions, indent=2) + "\n").encode() + changed = [ + { + "scenario_id": k[0], + "variable": k[1], + "frozen": base_values[k], + "previous": base_values[k], + "regenerated": float(v), + "cause": "mock", + "basis": "MOCK basis.", + } + for k, v in sorted(moved.items()) + ] + revision = { + "date": MOCK_DATE, + "kind": "engine_upgrade", + "root_cause": "engine_upgrade_policyengine_us_" + number.replace(".", "_"), + "outputs": "every scored output", + "rule": "MOCK rule.", + "engine_version": engine, + "previous_engine_version": driver.BASE_ENGINE, + "policyengine_py": "MOCK", + "fix_modules": [], + "builder": "MOCK", + "provenance": { + "actions_sha256": sha(actions_blob), + "base_commit": driver.BASE_COMMIT, + }, + "excluded_outputs_untouched": True, + "kept_exclusions_from_release": kept_ruled, + "excluded_outputs_rechecked": [ + { + "scenario_id": k[0], + "variable": k[1], + "kept_value": base_values[k], + "value_on_" + number.replace(".", "_"): v, + "reason": "MOCK reason.", + } + for k, v in sorted(rechecked.items()) + ], + "regenerated_exclusions": [ + { + "scenario_id": k[0], + "variable": k[1], + "target": targets.get( + k, + { + "kind": "record", + "value": float(records[k]["alternative_value"]), + "engine": records[k]["engine_version"], + }, + ), + "regenerated": float(v), + "tolerance": 1.0, + "upstream": "MOCK upstream fix", + "basis": "MOCK basis.", + "record": records[k], + } + for k, v in sorted(regenerated.items()) + ], + "new_exclusions": [ + {"scenario_id": k[0], "variable": k[1]} for k in sorted(added) + ], + "changed": changed, + } + meta = json.loads(base_files[META_NAME]) + meta["policyengine_bundles"]["us"]["model_version"] = number + meta["reference_csv_sha256"] = sha(csv_blob) + meta["regenerated_at_utc"] = f"{MOCK_DATE}T18:00:00+00:00" + meta["revisions"].append(revision) + out = root / "out" + out.mkdir(parents=True) + (out / CSV_NAME).write_bytes(csv_blob) + (out / META_NAME).write_text(json.dumps(meta, indent=2) + "\n") + (out / EXCLUSIONS_NAME).write_text(driver.exclusions_text(doc)) + traces = { + f"{item['scenario_id']}|{item['variable']}": { + "pe_variable": item["variable"], + "trace": "MOCK trace", + } + for item in changed + } + (out / TRACES_NAME).write_text(json.dumps(traces, indent=1)) + explanations = root / driver.EXPLANATIONS_NAME + explanations.write_bytes(explanations_with(base_explanations, moved)) + actions_path = root / driver.ACTIONS_NAME + actions_path.write_bytes(actions_blob) + return SimpleNamespace( + built=out, + explanations=explanations, + actions=actions_path, + moved=moved, + root=root, + ) + + +@functools.cache +def _real_baseline_files() -> tuple: + files = {name: base_blob(SNAPSHOT_PATH / name) for name in driver.UPGRADED_FILES} + return files, base_blob(EXPLANATIONS_PATH) + + +def upgrade_spec_dict() -> dict: + """The real spec without decisions on the upgrade's new exclusions or + triage: a mock build decides its own (upgraded_added). The release rules + nothing, so no ruled record is regenerated (no regenerated_by_upgrade).""" + spec = real_spec() + spec.pop("upgrade_adjudications", None) + spec.pop("triage_adjudications", None) + spec.pop("regenerated_by_upgrade", None) + return spec + + +@pytest.fixture +def upgrade_spec(monkeypatch): + """The spec as upgrade_spec_dict gives it.""" + spec = upgrade_spec_dict() + monkeypatch.setattr(driver, "load_spec", lambda: copy.deepcopy(spec)) + return spec + + +def real_build(root: Path, **changes) -> SimpleNamespace: + """MOCK: a build on release 20261010's real references: one approved move, + two new county exclusions and four 20261010 engine-defect records + regenerated, unless ``changes`` says otherwise.""" + files, explanations = _real_baseline_files() + options = { + "approved": MOCK_APPROVED, + "regenerated": MOCK_REGENERATED, + "added": {k: mock_new_record(k, v) for k, v in MOCK_NEW.items()}, + } + options.update(changes) + return write_build( + root, + files, + explanations, + json.loads(release_text()), + set(), + **options, + ) + + +def load(build: SimpleNamespace, baseline=None): + return driver.load_build( + build.built, build.explanations, build.actions, baseline=baseline + ) + + +def edit_json(path: Path, change, *, indent=2) -> None: + value = json.loads(path.read_text()) + change(value) + path.write_text(json.dumps(value, indent=indent) + "\n") + + +def repin_csv(build: SimpleNamespace) -> None: + """Rebind the sidecar to the build's (edited) reference CSV.""" + edit_json( + build.built / META_NAME, + lambda meta: meta.update(reference_csv_sha256=sha(build.built / CSV_NAME)), + ) + + +def last_revision(meta: dict) -> dict: + return meta["revisions"][-1] + + +def test_the_upgrade_constants_name_release_20261010s_files(): + """The pins an upgrade is gated against are release 20261010's, from git.""" + assert driver.UPGRADED_FILES == (CSV_NAME, META_NAME, EXCLUSIONS_NAME) + assert set(driver.UPGRADED_FILES) < set(driver.REFERENCE_FILES) + assert driver.BASE_EXPLANATIONS_SHA256 == sha(base_blob(EXPLANATIONS_PATH)) + assert driver.base_explanations_bytes() == base_blob(EXPLANATIONS_PATH) + for name in driver.UPGRADED_FILES: + assert driver.base_reference_bytes(name) == base_blob(SNAPSHOT_PATH / name) + meta = json.loads(base_blob(SNAPSHOT_PATH / META_NAME)) + assert last_revision(meta)["kind"] == "engine_upgrade" + assert last_revision(meta)["engine_version"] == driver.BASE_ENGINE + + +# A real build_references_upgrade.py build, its narratives and its actions, on +# the newest policyengine-us (local only, the release's build directory): the +# driver's gates read what the real builder writes, not only the mock build. +REAL_BUILD = REPO / "results/local/musewonder/engine/final" + + +def test_a_real_builders_build_passes_the_drivers_gates(): + """Differential: the builder and the driver agree on the build's shape. + The release's build regenerates the release 20261010 engine-defect records + its engine fixes, rules nothing, and the spec decides any new exclusion.""" + build = REAL_BUILD / "build" + if not (build / META_NAME).is_file(): + pytest.skip("needs the release's local reference build") + upgrade = driver.load_build( + build, + REAL_BUILD / driver.EXPLANATIONS_NAME, + REAL_BUILD / driver.ACTIONS_NAME, + ) + assert upgrade.previous_engine_version == driver.BASE_ENGINE + assert upgrade.regenerated_ruled == frozenset() + assert upgrade.regenerated_base + assert upgrade.records == 58 - len(upgrade.regenerated_base) + len(upgrade.added) + added = driver.upgrade_decisions(upgrade) + assert [driver.spec_key(item) for item, _ in added] == sorted(upgrade.added) + + +def test_a_build_on_release_20261010_passes_every_gate(upgrade_spec, tmp_path): + """MOCK build: one approved move, two new exclusions, four 20261010 + engine-defect records regenerated. 58 - 4 + 2 = 56 records, 1,928 scored + outputs.""" + build = real_build(tmp_path) + upgrade = load(build) + assert upgrade.engine_version == MOCK_ENGINE + assert upgrade.previous_engine_version == driver.BASE_ENGINE + assert set(upgrade.changed) == {*MOCK_APPROVED, *MOCK_NEW, *MOCK_REGENERATED} + assert upgrade.changed == pytest.approx(build.moved) + assert upgrade.regenerated_ruled == frozenset() + assert upgrade.regenerated_base == frozenset(MOCK_REGENERATED) + assert upgrade.added == frozenset(MOCK_NEW) + assert upgrade.records == 56 and upgrade.scored_outputs == 1928 + assert upgrade.sha256 == { + **{name: sha(build.built / name) for name in driver.BUILT_FILES}, + driver.EXPLANATIONS_NAME: sha(build.explanations), + driver.ACTIONS_NAME: sha(build.actions), + } + assert upgrade.upgraded_cases == {driver.output_case(k) for k in build.moved} + assert upgrade.dropped_cases == MOCK_REGENERATED_CASES + assert upgrade.exclusions_text == (build.built / EXCLUSIONS_NAME).read_text() + + +def test_a_build_may_regenerate_a_20261010_engine_defect(upgrade_spec, tmp_path): + """WI 042 lands on its record's alternative ($0): its record leaves, and + its release 20261010 decision is one adjudicate-exclusions drops.""" + regenerated = {**MOCK_REGENERATED, WI_042: 0.4} + upgrade = load(real_build(tmp_path, regenerated=regenerated)) + assert upgrade.regenerated_base == frozenset(regenerated) + assert upgrade.regenerated == frozenset(regenerated) + assert upgrade.records == 55 + assert upgrade.dropped_cases == MOCK_REGENERATED_CASES | {WI_042_CASE} + + +def test_a_build_that_regenerates_nothing_keeps_release_20261010s_record( + upgrade_spec, tmp_path +): + kept = load(real_build(tmp_path, regenerated={})) + assert kept.regenerated == frozenset() and kept.records == 60 + + +def _edit_revision(change): + return lambda meta: change(last_revision(meta)) + + +SIDECAR_TAMPERS = { + "an_earlier_revision": ( + lambda meta: meta["revisions"][0].update(date="2026-09-21"), + "keep release 20261010's revisions and add one", + ), + "two_revisions": ( + lambda meta: meta["revisions"].append(copy.deepcopy(meta["revisions"][-1])), + "keep release 20261010's revisions and add one", + ), + "not_an_upgrade": ( + _edit_revision(lambda r: r.update(kind="convention")), + "is not an engine_upgrade", + ), + "from_another_engine": ( + _edit_revision( + lambda r: r.update(previous_engine_version="policyengine-us 2.38.5") + ), + "does not upgrade from policyengine-us 2.38.6", + ), + "to_the_same_engine": ( + _edit_revision(lambda r: r.update(engine_version=driver.BASE_ENGINE)), + "names no newer engine", + ), + "unpinned_csv": ( + lambda meta: meta.update(reference_csv_sha256="0" * 64), + "does not pin the built reference CSV", + ), + "another_field": ( + lambda meta: meta.update(seed=43), + r"changes \['seed'\]", + ), + "another_bundle": ( + lambda meta: meta["policyengine_bundles"]["us"].update(model_version="2.37.0"), + "bundle does not name", + ), + "touched_exclusions": ( + _edit_revision(lambda r: r.update(excluded_outputs_untouched=False)), + "keep every excluded output untouched", + ), + "provenance": ( + _edit_revision(lambda r: r["provenance"].update(base_commit="0" * 40)), + "provenance does not name these actions", + ), +} + + +@pytest.mark.parametrize("tamper", sorted(SIDECAR_TAMPERS)) +def test_a_sidecar_that_is_not_20261010s_plus_one_upgrade_is_refused( + upgrade_spec, tmp_path, tamper +): + build = real_build(tmp_path) + change, problem = SIDECAR_TAMPERS[tamper] + edit_json(build.built / META_NAME, change) + with pytest.raises(SystemExit, match=problem): + load(build) + + +def _bump(build, key, delta): + base = (build.built / CSV_NAME).read_bytes() + values = driver.reference_values(base) + (build.built / CSV_NAME).write_bytes(csv_with(base, {key: values[key] + delta})) + repin_csv(build) + + +def _unlist(build, key): + edit_json( + build.built / META_NAME, + _edit_revision( + lambda r: r.update( + changed=[ + c for c in r["changed"] if (c["scenario_id"], c["variable"]) != key + ] + ) + ), + ) + + +def _list_unmoved(build, key, value): + entry = { + "scenario_id": key[0], + "variable": key[1], + "frozen": value, + "previous": value, + "regenerated": value, + "cause": "mock", + "basis": "MOCK", + } + edit_json( + build.built / META_NAME, + _edit_revision(lambda r: r["changed"].append(entry)), + ) + + +def _impact_weight(build): + lines = (build.built / CSV_NAME).read_text().splitlines(keepends=True) + fields = next(csv.reader([lines[5]])) + fields[3] = str(float(fields[3] or 0) + 1) + buffer = io.StringIO() + csv.writer(buffer, lineterminator="\n").writerow(fields) + lines[5] = buffer.getvalue() + (build.built / CSV_NAME).write_text("".join(lines)) + repin_csv(build) + + +def _swap_rows(build): + lines = (build.built / CSV_NAME).read_text().splitlines(keepends=True) + lines[1], lines[2] = lines[2], lines[1] + (build.built / CSV_NAME).write_text("".join(lines)) + repin_csv(build) + + +SNAP_000 = ("scenario_000", "snap") +CSV_TAMPERS = { + "an_unlisted_move": ( + lambda b: _bump(b, ("scenario_000", "payroll_tax"), 25.0), + "outside the build's changed list: unlisted moves", + ), + "an_unlisted_cent": ( + lambda b: _bump(b, ("scenario_000", "payroll_tax"), 0.01), + "outside the build's changed list: unlisted moves", + ), + "a_change_dropped_from_the_list": ( + lambda b: _unlist(b, next(iter(MOCK_APPROVED))), + "outside the build's changed list: unlisted moves", + ), + "a_listed_change_that_did_not_happen": ( + lambda b: _list_unmoved( + b, + ("scenario_000", "payroll_tax"), + driver.reference_values(_real_baseline_files()[0][CSV_NAME])[ + ("scenario_000", "payroll_tax") + ], + ), + "listed changes that did not happen", + ), + "a_misstated_value": ( + lambda b: edit_json( + b.built / META_NAME, + _edit_revision(lambda r: r["changed"][0].update(regenerated=1.0)), + ), + "misstates the previous or regenerated value", + ), + "an_impact_weight": (_impact_weight, "reference impact_weight changed"), + "two_rows_swapped": (_swap_rows, "outputs or their order changed"), +} + + +@pytest.mark.parametrize("tamper", sorted(CSV_TAMPERS)) +def test_a_reference_csv_that_differs_outside_the_changed_list_is_refused( + upgrade_spec, tmp_path, tamper +): + build = real_build(tmp_path) + change, problem = CSV_TAMPERS[tamper] + change(build) + with pytest.raises(SystemExit, match=problem): + load(build) + + +@functools.cache +def _scored_unmoved_keys() -> tuple: + """Release 20261010's outputs the mock build neither moves nor excludes.""" + files, _ = _real_baseline_files() + excluded = {driver.spec_key(r) for r in json.loads(release_text())["exclusions"]} + moved = {*MOCK_APPROVED, *MOCK_NEW, *MOCK_REGENERATED} + return tuple( + sorted(set(driver.reference_values(files[CSV_NAME])) - excluded - moved) + ) + + +@functools.cache +def _real_baseline(): + with mock.patch.object(driver, "load_spec", lambda: upgrade_spec_dict()): + return driver.upgrade_baseline() + + +@settings(max_examples=40, deadline=None) +@given( + index=st.integers(min_value=0, max_value=10**6), + delta=st.one_of( + st.floats(min_value=1e-6, max_value=1e5), + st.floats(min_value=-1e5, max_value=-1e-6), + ), + listed=st.booleans(), +) +def test_any_tampered_reference_or_extra_changed_entry_is_refused(index, delta, listed): + """For any output the build leaves alone: moving its value by any amount + without listing it is refused, and listing it without moving it is + refused.""" + keys = _scored_unmoved_keys() + key = keys[index % len(keys)] + with tempfile.TemporaryDirectory() as scratch: + build = real_build(Path(scratch)) + if listed: + value = driver.reference_values((build.built / CSV_NAME).read_bytes())[key] + _list_unmoved(build, key, value) + problem = "listed changes that did not happen" + else: + _bump(build, key, delta) + problem = "unlisted moves" + with pytest.raises(SystemExit, match=problem): + load(build, baseline=_real_baseline()) + + +def _edit_record(build, change): + edit_json(build.built / EXCLUSIONS_NAME, change) + + +def _record_in(doc, key): + return next(r for r in doc["exclusions"] if driver.spec_key(r) == key) + + +EXCLUSION_TAMPERS = { + "a_20261010_record_edited": ( + lambda b: _edit_record( + b, lambda doc: _record_in(doc, WI_042).update(note="Rewritten.") + ), + "changes release 20261010's record", + ), + "a_record_nobody_declared": ( + lambda b: _edit_record( + b, + lambda doc: doc["exclusions"].insert( + 0, mock_new_record(("scenario_000", "payroll_tax"), 1.0) + ), + ), + "not its actions'", + ), + "a_20261010_record_dropped": ( + lambda b: _edit_record( + b, + lambda doc: doc.update( + exclusions=[ + r for r in doc["exclusions"] if driver.spec_key(r) != WI_042 + ] + ), + ), + # No reviewed action regenerates it, so the build may not drop its record. + "actions do not regenerate it exactly once", + ), + "the_tail_moved": ( + lambda b: _edit_record( + b, + lambda doc: doc["exclusions"].insert(0, doc["exclusions"].pop()), + ), + "is not last", + ), + "records_reordered": ( + lambda b: _edit_record( + b, + lambda doc: doc["exclusions"].insert(1, doc["exclusions"].pop(0)), + ), + "changed their order", + ), + "another_form": ( + lambda b: edit_json(b.built / EXCLUSIONS_NAME, lambda doc: None, indent=1), + "not in the builder's form", + ), + "another_rule": ( + lambda b: _edit_record(b, lambda doc: doc.update(rule="Another rule.")), + "outside its records and derivation", + ), + "a_new_record_on_the_old_engine": ( + lambda b: _edit_record( + b, + lambda doc: _record_in(doc, next(iter(MOCK_NEW))).update( + engine_version=driver.BASE_ENGINE + ), + ), + "not computed on policyengine-us 2.40.9", + ), +} + + +@pytest.mark.parametrize("tamper", sorted(EXCLUSION_TAMPERS)) +def test_an_exclusion_record_other_than_the_releases_is_refused( + upgrade_spec, tmp_path, tamper +): + build = real_build(tmp_path) + change, problem = EXCLUSION_TAMPERS[tamper] + change(build) + with pytest.raises(SystemExit, match=problem): + load(build) + + +def _rebind_actions(build: SimpleNamespace) -> None: + """MOCK: point the build's revision provenance at its (edited) actions.""" + edit_json( + build.built / META_NAME, + lambda meta: last_revision(meta)["provenance"].update( + actions_sha256=sha(build.actions) + ), + ) + + +def test_every_scored_move_beyond_a_dollar_needs_an_approval(upgrade_spec, tmp_path): + """MOCK: the build moves ID 076 by $10; without its approval in the + actions (and the provenance rebound to match), the build is refused.""" + build = real_build(tmp_path) + load(build) + edit_json(build.actions, lambda plan: plan.update(approved=[])) + _rebind_actions(build) + with pytest.raises(SystemExit, match="without an approval"): + load(build) + + +ID_007 = ("scenario_007", STATE_TAX) + + +def _set_regeneration(build, key, *, action=None, revision=None) -> None: + """MOCK: edit one regeneration's action and/or revision entry, and point + the revision's provenance at the edited actions.""" + + def edit(items, fields): + for item in items: + if driver.spec_key(item) == key: + item.update(fields) + + if action is not None: + edit_json( + build.actions, lambda plan: edit(plan["regenerated_exclusions"], action) + ) + if revision is not None: + edit_json( + build.built / META_NAME, + lambda meta: edit(last_revision(meta)["regenerated_exclusions"], revision), + ) + _rebind_actions(build) + + +@pytest.mark.parametrize("tolerance, ok", [(1.0, True), (0.01, True), (0.001, False)]) +def test_a_regeneration_lands_within_its_own_tolerance( + upgrade_spec, tmp_path, tolerance, ok +): + """MOCK: ID 007 regenerates at 651.32 against its record's 651.317017; + a regeneration whose action allows $0.001 is held to it.""" + build = real_build(tmp_path) + _set_regeneration( + build, + ID_007, + action={"tolerance": tolerance}, + revision={"tolerance": tolerance}, + ) + if ok: + load(build) + else: + with pytest.raises(SystemExit, match=r"not within \$0.001"): + load(build) + + +def test_the_actions_tolerance_binds_not_the_revisions(upgrade_spec, tmp_path): + """MOCK, the delta review's case: the reviewed action allows ID 007 $0.001, + the revision keeps $1 and the provenance names the actions. The build is + refused: the revision does not restate its action.""" + build = real_build(tmp_path) + _set_regeneration(build, ID_007, action={"tolerance": 0.001}) + with pytest.raises(SystemExit, match="tolerance 1.0 is not the action's 0.001"): + load(build) + + +@pytest.mark.parametrize( + "action, revision, problem", + [ + ({"tolerance": 1.5}, {"tolerance": 1.5}, "tolerance 1.5 is not in"), + ({"tolerance": float("nan")}, {}, "tolerance nan is not in"), + ({"tolerance": "1"}, {}, "tolerance '1' is not in"), + ({}, {"tolerance": 0.5}, "tolerance 0.5 is not the action's 1.0"), + ({}, {"upstream": "MOCK other fix"}, "revision's upstream is not the action's"), + ({}, {"basis": "MOCK other basis."}, "revision's basis is not the action's"), + ({"alternative_value": 1.0}, {}, "alternative_value is not the record's"), + ( + {"target": {"kind": "fix_modules"}}, + {}, + "target is not the kind the action asks", + ), + ], +) +def test_a_regeneration_must_restate_its_reviewed_action( + upgrade_spec, tmp_path, action, revision, problem +): + build = real_build(tmp_path) + _set_regeneration(build, ID_007, action=action, revision=revision) + with pytest.raises(SystemExit, match=problem): + load(build) + + +def test_a_regeneration_needs_exactly_one_action(upgrade_spec, tmp_path): + build = real_build(tmp_path) + + def drop(plan): + plan["regenerated_exclusions"] = [ + item + for item in plan["regenerated_exclusions"] + if driver.spec_key(item) != ID_007 + ] + + edit_json(build.actions, drop) + _rebind_actions(build) + with pytest.raises(SystemExit, match="do not regenerate it exactly once|names"): + load(build) + + +def _evidence_target(key, modules, *, after_offset: float) -> tuple[dict, float]: + """A fix_modules target from the committed 2.37.2 evidence, as the builder + writes it, with the modules moving the build's value by ``after_offset`` + on a later engine. Returns the target and the build's value.""" + path = "reference_audit/2026-10-09-engine-upgrade/evidence/pe2.37.2.json" + doc = json.loads((REPO / path).read_text()) + (item,) = [ + i for i in doc["items"] if driver.spec_key(i) == key and i["modules"] == modules + ] + + def pin(name): + blob = driver.base_commit_blob(driver.AUDIT_FIXES / name) + return {"module": name, "sha256": hashlib.sha256(blob).hexdigest()} + + value = float(item["corrected_value"]) + return { + "kind": "fix_modules", + "value": value, + "engine": doc["engine"], + "engine_value": float(item["engine_value"]), + "modules": [pin(m) for m in modules], + "dependencies": [pin(d) for d in driver.committed_dependencies(modules)], + "evidence": path, + "evidence_sha256": sha(REPO / path), + "value_with_modules": value + after_offset, + }, value + + +def test_a_target_module_whose_closure_cannot_be_established_is_refused(): + """The delta review's case: a fix_modules target naming + c13v3_upstream_plus_r30.py, which loads its siblings through + _HERE / f"{name}.py", is refused, not pinned without them.""" + from policybench.fix_module_closure import ClosureError + + with pytest.raises(ClosureError, match="cannot be established"): + driver.committed_dependencies(["c13v3_upstream_plus_r30.py"]) + key = ("scenario_003", "federal_income_tax_before_refundable_credits") + target, value = _evidence_target( + key, ["r01_ira_compensation_v2.py"], after_offset=0 + ) + target["modules"].append( + {"module": "c13v3_upstream_plus_r30.py", "sha256": "0" * 64} + ) + problems = driver.regeneration_target_problems( + key, {}, target, value, "policyengine-us 2.99.0", 1.0 + ) + assert any( + "c13v3_upstream_plus_r30.py loads cannot be established" in p for p in problems + ) + + +@pytest.mark.parametrize("tolerance, ok", [(1.0, True), (0.25, False)]) +def test_module_inertness_uses_the_regenerations_tolerance(tolerance, ok): + """Committed evidence: on 2.37.2 the IRA compensation fix moves scenario + 003's federal tax. On a later engine it moves the build's value by $0.50: + inert within $1, not within the action's $0.25.""" + key = ("scenario_003", "federal_income_tax_before_refundable_credits") + target, value = _evidence_target( + key, ["r01_ira_compensation_v2.py"], after_offset=0.5 + ) + problems = driver.regeneration_target_problems( + key, {}, target, value, "policyengine-us 2.99.0", tolerance + ) + if ok: + assert problems == [] + else: + assert problems == [ + f"{key}: on policyengine-us 2.99.0 the audited fix still moves it: " + f"{value} -> {value + 0.5}" + ] + + +def test_a_regenerated_record_off_its_alternative_is_refused(upgrade_spec, tmp_path): + """WI 042's record says $0; the build regenerates it at $1.50.""" + regenerated = {**MOCK_REGENERATED, WI_042: 1.5} + with pytest.raises(SystemExit, match=r"not within \$1 of its audited target"): + load(real_build(tmp_path, regenerated=regenerated)) + + +def test_only_an_engine_defect_can_be_regenerated(upgrade_spec, tmp_path): + """An unlisted input stays unlisted on any engine: scenario_007's + Medicare record cannot leave, even on its alternative value.""" + regenerated = {**MOCK_REGENERATED, MEDICARE_007: 1.0} + with pytest.raises(SystemExit, match="regenerates only an engine defect"): + load(real_build(tmp_path, regenerated=regenerated)) + + +def test_an_excluded_output_keeps_the_value_it_was_decided_on(upgrade_spec, tmp_path): + """Rule 5: a kept record's output moved (and listed) is refused.""" + build = real_build(tmp_path, approved={**MOCK_APPROVED, WI_042: 290.0}) + with pytest.raises(SystemExit, match="keeps the value it was decided on"): + load(build) + + +@pytest.mark.parametrize( + "outputs, problem", + [ + ([["scenario_042"]], "is not a list of"), + ("scenario_042", "is not a list of"), + ([list(WI_042)], "does not rule on"), + ([["scenario_000", "snap"]], "does not rule on"), + ( + [ + ["scenario_042", STATE_TAX], + ["scenario_042", STATE_TAX], + ], + "names an output twice", + ), + ], +) +def test_the_specs_regenerated_section_can_name_no_output(outputs, problem): + """The release rules nothing, so regenerated_by_upgrade may name no + output: a release 20261010 record is regenerated through the build's + actions, never through the spec.""" + spec = {**real_spec(), "regenerated_by_upgrade": {"outputs": outputs}} + with mock.patch.object(driver, "ROOT", REPO): + with pytest.raises(SystemExit, match=problem): + driver.spec_regenerated(spec) + + +def test_the_specs_regenerated_section_is_read_as_written(): + assert "regenerated_by_upgrade" not in real_spec() + assert driver.spec_regenerated(real_spec() | {"x": 1}) == frozenset() + assert driver.spec_regenerated(upgrade_spec_dict()) == frozenset() + + +@settings(max_examples=60) +@given( + variable=st.sampled_from( + ["state_income_tax_before_refundable_credits", "snap", "head_medicaid_eligible"] + ), + alternative=st.floats(min_value=0, max_value=1e5), + delta=st.floats(min_value=-3, max_value=3), +) +def test_a_regeneration_lands_within_a_dollar_or_on_the_flag( + variable, alternative, delta +): + """An amount lands within $1 of the alternative; a flag lands only on it.""" + if variable.endswith("_eligible"): + alternative = float(round(alternative) % 2) + value = float(round(alternative + delta) % 2) + assert driver.regeneration_lands(variable, value, alternative) == ( + value == alternative + ) + else: + value = alternative + delta + assert driver.regeneration_lands(variable, value, alternative) == ( + abs(value - alternative) <= 1.0 + ) + + +ENGINE_DEFECTS = tuple( + sorted( + driver.spec_key(r) + for r in json.loads(base_blob(SNAPSHOT_PATH / EXCLUSIONS_NAME))["exclusions"] + if r["reason_code"] == "reference_engine_defect" + ) +) +NEW_POOL = (*MOCK_NEW, ("scenario_000", "payroll_tax")) + + +@settings(max_examples=30, deadline=None) +@given( + regenerate_base=st.sets(st.sampled_from(ENGINE_DEFECTS), max_size=6), + add=st.sets(st.sampled_from(NEW_POOL)), + offset=st.floats(min_value=-0.9, max_value=0.9), + mutation=st.sampled_from(["none", "drop_kept", "add_undeclared"]), +) +def test_the_release_record_is_exactly_base_less_regenerated_plus_new( + regenerate_base, add, offset, mutation +): + """For any regenerated 20261010 engine defects and any new exclusions: the + gate accepts the build exactly when its record is 20261010's records less + the regenerated, plus the new; and then it names each set. Dropping a kept + record or adding an undeclared one is refused.""" + files, explanations = _real_baseline_files() + baseline = _real_baseline() + assert baseline.ruled == {} and baseline.regenerated == frozenset() + release = json.loads(release_text()) + records = {driver.spec_key(r): r for r in release["exclusions"]} + values = driver.reference_values(files[CSV_NAME]) + regenerated = { + key: float(records[key]["alternative_value"]) + offset + for key in regenerate_base + } + added = {key: mock_new_record(key, values[key] + 50.0) for key in add} + with tempfile.TemporaryDirectory() as scratch: + build = write_build( + Path(scratch), + files, + explanations, + release, + set(), + approved={}, + regenerated=regenerated, + added=added, + ) + expected = (set(records) - set(regenerated)) | set(added) + if mutation == "none": + upgrade = load(build, baseline=baseline) + got = { + driver.spec_key(r) + for r in json.loads(upgrade.exclusions_text)["exclusions"] + } + assert got == expected + assert upgrade.regenerated_base == frozenset(regenerate_base) + assert upgrade.regenerated_ruled == frozenset() + assert upgrade.added == frozenset(add) + assert upgrade.records == 58 - len(regenerated) + len(added) + return + if mutation == "drop_kept": + victim = sorted(expected - set(added) - {TAIL})[0] + _edit_record( + build, + lambda doc: doc.update( + exclusions=[ + r for r in doc["exclusions"] if driver.spec_key(r) != victim + ] + ), + ) + else: + _edit_record( + build, + lambda doc: doc["exclusions"].insert( + 0, mock_new_record(("scenario_001", "payroll_tax"), 1.0) + ), + ) + with pytest.raises(SystemExit): + load(build, baseline=baseline) + + +def test_the_traces_must_cover_exactly_the_changed_outputs(upgrade_spec, tmp_path): + build = real_build(tmp_path) + traces = json.loads((build.built / TRACES_NAME).read_text()) + traces.pop(sorted(traces)[0]) + (build.built / TRACES_NAME).write_text(json.dumps(traces)) + with pytest.raises(SystemExit, match="traces do not cover exactly"): + load(build) + + +def _explanation_rows(build): + return list(csv.reader(io.StringIO(build.explanations.read_text(), newline=""))) + + +def _write_explanations(build, rows): + buffer = io.StringIO() + csv.writer(buffer, lineterminator="\n").writerows(rows) + build.explanations.write_text(buffer.getvalue()) + + +def _explain(build, key, column, text): + rows = _explanation_rows(build) + header = rows[0] + for row in rows[1:]: + if (row[1], row[2]) == key: + row[header.index(column)] = text + _write_explanations(build, rows) + + +EXPLANATION_TAMPERS = { + "an_unchanged_row_reworded": ( + lambda b: _explain(b, SNAP_000, "explanation", "Reworded."), + "rewrite outputs it did not change", + ), + "a_stale_value": ( + lambda b: _explain(b, next(iter(MOCK_APPROVED)), "reference_value", "6828.34"), + "do not state the new reference", + ), + "an_empty_narrative": ( + lambda b: _explain(b, next(iter(MOCK_APPROVED)), "explanation", " "), + "do not state the new reference", + ), + "rows_swapped": ( + lambda b: _write_explanations( + b, [(rows := _explanation_rows(b))[0], rows[2], rows[1], *rows[3:]] + ), + "outputs or their order changed", + ), + "a_column_renamed": ( + lambda b: _write_explanations( + b, + [ + [ + "narrative" if c == "explanation" else c + for c in _explanation_rows(b)[0] + ], + *_explanation_rows(b)[1:], + ], + ), + "columns changed", + ), +} + + +@pytest.mark.parametrize("tamper", sorted(EXPLANATION_TAMPERS)) +def test_explanations_must_differ_from_20261010s_only_on_changed_outputs( + upgrade_spec, tmp_path, tamper +): + build = real_build(tmp_path) + change, problem = EXPLANATION_TAMPERS[tamper] + change(build) + with pytest.raises(SystemExit, match=problem): + load(build) + + +ACTION_TAMPERS = { + "another_engine": ( + lambda plan: plan.update(engine="policyengine-us 2.41.0"), + "actions name 'policyengine-us 2.41.0'", + ), + "an_approved_value_off_by_a_cent": ( + lambda plan: plan["approved"][0].update(value=6838.35), + "approve moves the build did not make", + ), + "an_undeclared_new_record": ( + lambda plan: plan.update(new_exclusions=plan["new_exclusions"][:1]), + "new exclusions are not its actions'", + ), + "another_previous_engine": ( + lambda plan: plan.update(previous_engine="policyengine-us 2.38.5"), + "upgrade from 'policyengine-us 2.38.5'", + ), + "a_regeneration_left_out": ( + lambda plan: plan.update( + regenerated_exclusions=plan["regenerated_exclusions"][1:] + ), + # The regeneration is bound to its action before the lists are compared. + "actions do not regenerate it exactly once", + ), + "a_kept_ruled_output_named": ( + lambda plan: plan.update( + kept_exclusions_from_release=[ + {"scenario_id": WI_042[0], "variable": WI_042[1]} + ] + ), + "actions' kept_exclusions_from_release names", + ), +} + + +@pytest.mark.parametrize("tamper", sorted(ACTION_TAMPERS)) +def test_actions_that_do_not_describe_the_build_are_refused( + upgrade_spec, tmp_path, tamper +): + """Each edit also changes the actions' bytes, which the revision's + provenance pins; the refusal names the first disagreement.""" + build = real_build(tmp_path) + change, problem = ACTION_TAMPERS[tamper] + edit_json(build.actions, change) + pinned = sha(build.actions) + edit_json( + build.built / META_NAME, + _edit_revision(lambda r: r["provenance"].update(actions_sha256=pinned)), + ) + with pytest.raises(SystemExit, match=problem): + load(build) + + +def test_actions_edited_after_the_build_are_refused(upgrade_spec, tmp_path): + """The revision's provenance pins the actions file the build ran on.""" + build = real_build(tmp_path) + edit_json(build.actions, lambda plan: plan.update(review="An edit.")) + with pytest.raises(SystemExit, match="provenance does not name these actions"): + load(build) + + +def test_an_approved_value_within_half_a_cent_passes(upgrade_spec, tmp_path): + build = real_build(tmp_path) + edit_json(build.actions, lambda plan: plan["approved"][0].update(value=6838.344)) + pinned = sha(build.actions) + edit_json( + build.built / META_NAME, + _edit_revision(lambda r: r["provenance"].update(actions_sha256=pinned)), + ) + assert load(build).records == 56 + + +def test_a_rechecked_output_must_stay_excluded_at_its_value(upgrade_spec, tmp_path): + """MOCK: CA 005 federal moves on the mock engine and stays excluded; the + revision and the actions recheck it, keeping its 20261010 value.""" + key = ("scenario_005", "federal_income_tax_before_refundable_credits") + build = real_build(tmp_path, rechecked={key: 108525.578125}) + assert load(build).records == 56 + edit_json( + build.built / META_NAME, + _edit_revision( + lambda r: r["excluded_outputs_rechecked"][0].update(kept_value=1.0) + ), + ) + with pytest.raises(SystemExit, match="rechecks outputs it does not keep"): + load(build) + + +# --- Installing a build on a stage --------------------------------------------- + +TINY_META = { + "country": "us", + "policyengine_bundles": { + "us": {"model_package": "policyengine-us", "model_version": "2.15.17"} + }, + "reference_csv_sha256": None, + "revisions": [{"kind": "engine_upgrade", "engine_version": driver.BASE_ENGINE}], + "regenerated_at_utc": "2026-09-29T15:04:45+00:00", +} +TINY_RECORD = { + "schema_version": 1, + "rule": "MOCK rule.", + "derivation": "MOCK derivation.", + "exclusions": [], +} +EXPLANATION_FIELDS = [ + "country", + "scenario_id", + "variable", + "reference_value", + "trace_lines", + "explanation", + "error", +] + + +def _tiny_explanations() -> bytes: + """Release 20261010's stand-in explanations for the two-household board: + a row per output, with no narrative, so no seed prompt renders one.""" + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + writer.writerow(EXPLANATION_FIELDS) + for row in REFERENCE_ROWS: + writer.writerow( + ["us", row["scenario_id"], row["variable"], repr(row["value"]), 1, "", ""] + ) + return buffer.getvalue().encode() + + +def tiny_upgrade_stage(root: Path, stack: contextlib.ExitStack, haiku=None): + """A prepared stage on the two-household board (seeded_stage's), with + release 20261010 stood in by the board's own references: a sidecar, an + empty exclusion record and narrative-free explanations, bound in + stage.json. Muse Wonder misses s0 alone (JOINS_S0) unless ``haiku`` + says otherwise; s0's new verdict is written. Every patch goes on ``stack``. + """ + from policybench.audit import prepare_audit + + def patch(name, value): + stack.enter_context(mock.patch.object(driver, name, value)) + + patch("reworded_since_seed", lambda: frozenset()) + grounding = root / "grounding.csv" + pd.DataFrame( + [{"scenario_id": "s1", "variable": "snap", "grounding": "Gross test: pass."}] + ).to_csv(grounding, index=False) + patch("GROUNDING_SHA256", sha(grounding)) + lookup = {("s1", "snap"): "Gross test: pass."} + seed = root / "seed" + seed_board = _board(root / "seed-board", INCUMBENT_ROWS) + # The seed is on judge template v1, as release 20261010's was. + with driver.object_strings(): + prepare_audit( + seed_board / "us", seed, grounding_lookup=lookup, template_version=1 + ) + for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): + write_verdict( + seed / "cases" / item["case_id"], + _verdict(item["wrong_models"]), + judge_model_requested="default", + judge_model_reported=["gpt-6.1-sol"], + prompt_sha256=None, + judge_template_version=None, + ) + digest_text = driver.seed_digest_text(driver.seed_digest(seed)) + patch("SEED_DIGEST_SHA256", sha(digest_text.encode())) + stage = root / "stage" + bundle = _board( + stage / "publish" / driver.RUN_NAME, INCUMBENT_ROWS + (haiku or JOINS_S0) + ) + us = bundle / "us" + meta = {**TINY_META, "reference_csv_sha256": sha(us / CSV_NAME)} + (us / META_NAME).write_text(json.dumps(meta, indent=2) + "\n") + (us / EXCLUSIONS_NAME).write_text(driver.exclusions_text(TINY_RECORD)) + (bundle / "annotations").mkdir() + explanations = _tiny_explanations() + (bundle / "annotations" / driver.EXPLANATIONS_NAME).write_bytes(explanations) + (stage / "scoring").mkdir() + base = {name: (us / name).read_bytes() for name in driver.UPGRADED_FILES} + for name, blob in base.items(): + (stage / "scoring" / name).write_bytes(blob) + patch( + "BASE_REFERENCE_SHA256", + {**driver.BASE_REFERENCE_SHA256, **{n: sha(b) for n, b in base.items()}}, + ) + patch("BASE_EXPLANATIONS_SHA256", sha(explanations)) + patch("base_reference_bytes", lambda name: base[name]) + patch("base_explanations_bytes", lambda: explanations) + baseline = driver.Baseline(base, explanations, {}, frozenset()) + patch("upgrade_baseline", lambda spec=None: baseline) + patch("release_exclusions_sha256", lambda: "e" * 64) + args = SimpleNamespace(stage_dir=stage, audit_seed=seed, grounding=grounding) + with driver.object_strings(): + binding = driver.prepare_cases(args, bundle) + files = {f"publish/{driver.RUN_NAME}/us/{n}": sha(b) for n, b in base.items()} + (stage / "stage.json").write_text( + json.dumps({"partial": False, "early": False, "files": files, "seed": binding}) + ) + audit = stage / "audit" + if (audit / "cases/us__s0__snap").is_dir(): + write_verdict(audit / "cases/us__s0__snap", _verdict(["m1", "m2", NEW])) + counter = iter(range(10**6)) + + def build(approved, **options): + return write_build( + root / f"build-{next(counter)}", + base, + explanations, + TINY_RECORD, + (), + approved=approved, + **options, + ) + + def install(made): + driver.install_references( + SimpleNamespace( + stage_dir=stage, + built=made.built, + explanations=made.explanations, + actions=made.actions, + grounding=grounding, + ) + ) + + return SimpleNamespace( + stage=stage, + audit=audit, + seed=seed, + bundle=bundle, + base=base, + explanations=explanations, + grounding=grounding, + build=build, + install=install, + ) + + +@pytest.fixture +def upgrade_stage(tmp_path): + with contextlib.ExitStack() as stack: + yield tiny_upgrade_stage(tmp_path, stack) + + +def stage_files(stage: Path) -> dict: + return { + str(p.relative_to(stage)): p.read_bytes() + for p in sorted(stage.rglob("*")) + if p.is_file() + } + + +S0, S1, S2 = "us__s0__snap", "us__s1__snap", "us__s2__snap" +# s1's reference moves by 40 cents: every model's answer keeps its class (Claude +# Haiku 5.5's $300 still matches), but the prompt renders the new value and +# narrative, so the incumbent-only case is re-opened. +S1_MOVE = {("s1", "snap"): 300.4} + + +def test_install_references_reopens_only_the_cases_whose_prompts_change( + upgrade_stage, +): + s = upgrade_stage + assert json.loads((s.stage / driver.PROMPT_CHANGES).read_text()) == { + "added": [], + "changed": [S0], + "kept": [S1], + } + kept_verdict = (s.audit / "cases" / S0 / "verdict.json").read_bytes() + seed_verdict = (s.audit / "cases" / S1 / "verdict.json").read_bytes() + s.install(s.build(S1_MOVE)) + assert json.loads((s.stage / driver.PROMPT_CHANGES).read_text()) == { + "added": [], + "changed": [S0, S1], + "kept": [], + } + assert json.loads((s.stage / "pending.json").read_text()) == [S1] + # s0's prompt did not change, so its new verdict stays, byte for byte. + assert (s.audit / "cases" / S0 / "verdict.json").read_bytes() == kept_verdict + # s1's seed verdict is set aside with its reason, never deleted. + assert not (s.audit / "cases" / S1 / "verdict.json").exists() + (aside,) = (s.stage / "rejected-verdicts" / S1).iterdir() + assert (aside / "verdict.json").read_bytes() == seed_verdict + assert "install-references" in (aside / "reason.txt").read_text() + prompt = (s.audit / "cases" / S1 / "prompt.md").read_text() + assert "$300.40" in prompt and "MOCK narrative" in prompt + # The gates that re-derive the lists agree; s1 is re-opened by the upgrade. + assert driver.upgraded_cases(s.stage) == {S1} + assert driver.rejudged_cases(s.stage) == {S0, S1} + seed = driver.load_seed(s.stage) + assert driver.validate_verdicts(s.audit, seed=seed) == [S1] + write_verdict(s.audit / "cases" / S1, _verdict(["m2"])) + assert driver.validate_verdicts(s.audit, seed=seed) == [] + + +def test_install_references_installs_and_binds_the_build(upgrade_stage): + s = upgrade_stage + made = s.build(S1_MOVE) + s.install(made) + receipt = json.loads((s.stage / "stage.json").read_text()) + upgrade = driver.load_build(made.built, made.explanations, made.actions) + for name in driver.UPGRADED_FILES: + blob = (made.built / name).read_bytes() + assert (s.stage / "scoring" / name).read_bytes() == blob + assert (s.bundle / "us" / name).read_bytes() == blob + assert receipt["files"][f"publish/{driver.RUN_NAME}/us/{name}"] == sha(blob) + bound = f"publish/{driver.RUN_NAME}/annotations/{driver.EXPLANATIONS_NAME}" + assert receipt["files"][bound] == sha(made.explanations) + for name in driver.BUILD_COPY_FILES: + source = made.explanations if name == driver.EXPLANATIONS_NAME else None + source = made.actions if name == driver.ACTIONS_NAME else source + source = source or made.built / name + assert (s.stage / driver.BUILD_COPY / name).read_bytes() == source.read_bytes() + installed = receipt["references_installed"] + assert installed["engine_version"] == MOCK_ENGINE + assert installed["previous_engine_version"] == driver.BASE_ENGINE + assert installed["sha256"] == upgrade.sha256 + assert installed["actions_sha256"] == sha(made.actions) + assert installed["changed"] == [["s1", "snap"]] + assert installed["records"] == 0 and installed["scored_outputs"] == 1984 + assert installed["superseded"] == [] + assert receipt["exclusions_installed"] == { + "base_sha256": driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME], + "sha256": upgrade.sha256[EXCLUSIONS_NAME], + "spec_sha256": driver.digest(REPO / driver.SPEC_PATH), + "records": 0, + "source": f"{driver.BUILD_COPY}/{EXCLUSIONS_NAME}", + "engine_version": MOCK_ENGINE, + } + # Every binding the resume check reads holds, and the stage re-gates. + for name, pin in receipt["files"].items(): + assert driver.digest(s.stage / name) == pin + assert driver.stage_upgrade(s.stage) == upgrade + # BASE_REFERENCE_SHA256 stays the base's; the staged files are the build's. + assert driver.upgraded_reference_pins(upgrade)[CSV_NAME] == sha( + made.built / CSV_NAME + ) + pins = driver.upgraded_reference_pins(upgrade) + assert {name: pins[name] for name in driver.UPGRADED_FILES} == { + name: driver.digest(s.bundle / "us" / name) for name in driver.UPGRADED_FILES + } + assert pins["scenarios.csv"] == driver.BASE_REFERENCE_SHA256["scenarios.csv"] + + +def test_installing_the_same_build_again_changes_nothing(upgrade_stage): + s = upgrade_stage + made = s.build(S1_MOVE) + s.install(made) + once = stage_files(s.stage) + s.install(made) + assert stage_files(s.stage) == once + + +TINY_VALUES = { + "s0": st.sampled_from([0.0, 0.3, 50.0, 250.0]), + "s1": st.sampled_from([300.0, 300.4, 299.2, 310.0, 0.0]), + "s2": st.sampled_from([100.0, 100.5, 150.0]), +} + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given( + values=st.fixed_dictionaries(TINY_VALUES), haiku_s2=st.sampled_from([100.0, 150.0]) +) +def test_install_references_is_idempotent_for_any_build(values, haiku_s2): + """For any moves of the three references (and either answer of Claude + Haiku 5.5 on s2): installing the build gives a stage whose gates agree, + every verdict whose prompt kept its bytes keeps its bytes, and installing + it again changes no byte of the stage.""" + haiku = [ + (NEW, "s0", 99.0, "Guessed."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", haiku_s2, "Computed."), + ] + moves = { + (scenario, "snap"): value + for scenario, value in values.items() + if value != {"s0": 0.0, "s1": 300.0, "s2": 100.0}[scenario] + } + with tempfile.TemporaryDirectory() as scratch, contextlib.ExitStack() as stack: + s = tiny_upgrade_stage(Path(scratch), stack, haiku=haiku) + prompts = { + case.name: (case / "prompt.md").read_text() + for case in (s.audit / "cases").iterdir() + } + verdicts = { + case.name: (case / "verdict.json").read_bytes() + for case in (s.audit / "cases").iterdir() + if (case / "verdict.json").is_file() + } + made = s.build(moves) + try: + s.install(made) + except SystemExit as refusal: + # Only a seed case that leaves the audit may refuse: the moves + # cannot open an unexplained case. + assert "seed cases vanished" in str(refusal) + return + once = stage_files(s.stage) + s.install(made) + assert stage_files(s.stage) == once + assert driver.upgraded_cases(s.stage) == { + driver.output_case(key) for key in moves + } + driver.load_seed(s.stage) + for case, blob in verdicts.items(): + path = s.audit / "cases" / case + if path.is_dir() and (path / "prompt.md").read_text() == prompts[case]: + assert (path / "verdict.json").read_bytes() == blob + + +def test_a_move_that_makes_the_new_model_miss_reopens_the_case_with_it( + upgrade_stage, +): + """s1 moves to $310: Muse Wonder's $300 is now wrong, so the case + re-opens with it among the wrong models, as the predictions bear out.""" + s = upgrade_stage + s.install(s.build({("s1", "snap"): 310.0})) + manifest = { + item["case_id"]: item + for item in map(json.loads, (s.audit / "cases.jsonl").read_text().splitlines()) + } + assert NEW in manifest[S1]["wrong_models"] + assert driver.rejudged_cases(s.stage) == {S0, S1} + + +def test_a_case_the_upgrade_adds_without_the_new_model_is_reopened(tmp_path): + """s2's reference moves from $100 to $150, which Muse Wonder answered + and the incumbents did not: an incumbent-only case appears, which only the + upgrade explains, and its earlier Haiku-only verdict is set aside.""" + haiku = [ + (NEW, "s0", 99.0, "Guessed."), + (NEW, "s1", 300.0, "Right."), + (NEW, "s2", 150.0, "Computed."), + ] + with contextlib.ExitStack() as stack: + s = tiny_upgrade_stage(tmp_path, stack, haiku=haiku) + assert json.loads((s.stage / driver.PROMPT_CHANGES).read_text())["added"] == [ + S2 + ] + write_verdict(s.audit / "cases" / S2, _verdict([NEW])) + s.install(s.build({("s2", "snap"): 150.0})) + assert json.loads((s.stage / driver.PROMPT_CHANGES).read_text()) == { + "added": [S2], + "changed": [S0], + "kept": [S1], + } + manifest = { + item["case_id"]: item + for item in map( + json.loads, (s.audit / "cases.jsonl").read_text().splitlines() + ) + } + assert NEW not in manifest[S2]["wrong_models"] + assert json.loads((s.stage / "pending.json").read_text()) == [S2] + assert (s.stage / "rejected-verdicts" / S2).is_dir() + driver.load_seed(s.stage) + + +def test_the_upgrade_allowance_admits_only_the_cases_it_changed(): + """classify_prompt_changes: an incumbent-only changed or added case passes + only when the upgrade (or a rewording) changed its reference.""" + manifest = [ + {"case_id": S0, "wrong_models": ["m1"], "parse_failure_only": False}, + {"case_id": S1, "wrong_models": ["m1"], "parse_failure_only": False}, + ] + prompts = {S0: "a" * 64, S1: "b" * 64} + seeded = {S0: "c" * 64} + assert driver.classify_prompt_changes( + manifest, prompts, seeded, frozenset(), frozenset({S0, S1}) + ) == {"kept": [], "changed": [S0], "added": [S1]} + with pytest.raises(SystemExit, match=re.escape(f"[{S1!r}]")): + driver.classify_prompt_changes( + manifest, prompts, seeded, frozenset(), frozenset({S0}) + ) + with pytest.raises(SystemExit, match="incumbent-only case prompts changed"): + driver.classify_prompt_changes(manifest, prompts, seeded, frozenset()) + + +@settings(max_examples=150, deadline=None) +@given( + flags=st.lists( + st.fixed_dictionaries( + { + "seeded": st.booleans(), + "same_prompt": st.booleans(), + "new_wrong": st.booleans(), + "reworded": st.booleans(), + "upgraded": st.booleans(), + } + ), + max_size=7, + ) +) +def test_classify_prompt_changes_follows_its_rule_with_an_upgrade(flags): + """For any audit: a changed or added case is refused exactly when neither + the new model, a rewording nor the upgrade explains it.""" + manifest, prompts, seeded = [], {}, {} + reworded, upgraded = set(), set() + for index, f in enumerate(flags): + case = f"us__s{index}__snap" + wrong = ["m1", *([NEW] if f["new_wrong"] else [])] + manifest.append( + {"case_id": case, "wrong_models": wrong, "parse_failure_only": False} + ) + prompts[case] = f"{index:064d}" + if f["seeded"]: + seeded[case] = prompts[case] if f["same_prompt"] else "f" * 64 + if f["reworded"]: + reworded.add(case) + if f["upgraded"]: + upgraded.add(case) + offending = [ + item["case_id"] + for item, f in zip(manifest, flags) + if not (f["seeded"] and f["same_prompt"]) + and not f["new_wrong"] + and not f["reworded"] + and not f["upgraded"] + ] + unrendered = sorted( + c for c, f in zip(seeded, flags) if False + ) # no reworded case outside the manifest here + changed_or_added = { + item["case_id"] + for item, f in zip(manifest, flags) + if not (f["seeded"] and f["same_prompt"]) + } + unrendered = sorted(reworded - {c for c in changed_or_added if c in seeded}) + if offending: + with pytest.raises(SystemExit, match="incumbent-only case prompts changed"): + driver.classify_prompt_changes( + manifest, prompts, seeded, frozenset(reworded), frozenset(upgraded) + ) + elif unrendered: + with pytest.raises(SystemExit, match="reworded cases whose prompts"): + driver.classify_prompt_changes( + manifest, prompts, seeded, frozenset(reworded), frozenset(upgraded) + ) + else: + out = driver.classify_prompt_changes( + manifest, prompts, seeded, frozenset(reworded), frozenset(upgraded) + ) + assert set(out["changed"]) | set(out["added"]) == changed_or_added + + +def test_install_references_refuses_a_tampered_build_and_writes_nothing( + upgrade_stage, +): + s = upgrade_stage + made = s.build(S1_MOVE) + values = driver.reference_values((made.built / CSV_NAME).read_bytes()) + (made.built / CSV_NAME).write_bytes( + csv_with( + (made.built / CSV_NAME).read_bytes(), + {("s2", "snap"): values[("s2", "snap")] + 1}, + ) + ) + repin_csv(made) + before = stage_files(s.stage) + with pytest.raises(SystemExit, match="unlisted moves"): + s.install(made) + assert stage_files(s.stage) == before + + +def test_install_references_refuses_a_stage_off_the_base(upgrade_stage): + s = upgrade_stage + path = s.stage / "scoring" / CSV_NAME + path.write_bytes(path.read_bytes() + b"s9,snap,1.0\n") + before = stage_files(s.stage) + with pytest.raises(SystemExit, match="neither release 20261010's nor an"): + s.install(s.build(S1_MOVE)) + assert stage_files(s.stage) == before + + +def test_install_references_refuses_another_grounding(upgrade_stage): + s = upgrade_stage + s.grounding.write_text(s.grounding.read_text() + "s2,snap,Other.\n") + with pytest.raises(SystemExit, match="grounding differs"): + s.install(s.build(S1_MOVE)) + + +def test_a_second_build_replaces_the_first_and_is_recorded(upgrade_stage): + """The engine moves again (as the hub lands more fixes): the second build + replaces the first, gated against release 20261010 as the first was; the + receipt keeps the first under superseded; and going back to the first + build reinstalls it.""" + s = upgrade_stage + first = s.build(S1_MOVE) + s.install(first) + recorded = json.loads((s.stage / "stage.json").read_text())["references_installed"] + second = s.build( + {("s1", "snap"): 300.4, ("s2", "snap"): 100.5}, + engine="policyengine-us 2.38.0", + ) + s.install(second) + installed = json.loads((s.stage / "stage.json").read_text())["references_installed"] + assert installed["engine_version"] == "policyengine-us 2.38.0" + assert installed["superseded"] == [ + {k: v for k, v in recorded.items() if k != "superseded"} + ] + assert driver.upgraded_cases(s.stage) == {S1, S2} + driver.load_seed(s.stage) + assert driver.stage_upgrade(s.stage).engine_version == "policyengine-us 2.38.0" + + +def test_a_staged_reference_outside_the_revision_stops_every_seed_read( + upgrade_stage, +): + """upgraded_cases derives its cases from the staged files against the base + in git, so a staged CSV (or explanation) edited after the install stops + load_seed rather than widening what may re-open.""" + s = upgrade_stage + s.install(s.build(S1_MOVE)) + path = s.bundle / "us" / CSV_NAME + path.write_bytes(csv_with(path.read_bytes(), {("s2", "snap"): 101.5})) + with pytest.raises(SystemExit, match="outside the installed revision's changed"): + driver.load_seed(s.stage) + + +def test_later_steps_regate_the_installed_build(upgrade_stage): + s = upgrade_stage + s.install(s.build(S1_MOVE)) + copy_path = s.stage / driver.BUILD_COPY / driver.ACTIONS_NAME + copy_path.write_text(copy_path.read_text() + " ") + receipt = json.loads((s.stage / "stage.json").read_text()) + with pytest.raises(SystemExit, match="installed build's final_actions.json"): + driver.verify_installed_references(s.stage, receipt) + + +def test_install_exclusions_on_an_upgraded_stage_keeps_the_builds_record( + upgrade_stage, +): + """With an upgrade installed, install-exclusions re-gates the build and + binds its record; build_release_exclusions is never consulted.""" + s = upgrade_stage + made = s.build(S1_MOVE) + s.install(made) + once = stage_files(s.stage) + with mock.patch.object( + driver, "build_release_exclusions", lambda *a: pytest.fail("rebuilt") + ): + driver.install_exclusions(SimpleNamespace(stage_dir=s.stage)) + assert stage_files(s.stage) == once + + +def test_without_an_install_a_stage_has_no_upgrade(seeded_stage): + """The 2.15.17 path: nothing reads a build, nothing is derived from one.""" + _, stage, prepare = seeded_stage + prepare(JOINS_S0) + assert driver.stage_upgrade(stage) is None + assert driver.upgraded_cases(stage) == frozenset() + assert driver.stage_upgrade(stage / "no-such-stage") is None + + +@pytest.mark.parametrize( + "missing", ["--built", "--explanations", "--actions", "--grounding"] +) +def test_install_references_needs_the_build_its_explanations_actions_and_grounding( + tmp_path, capsys, missing +): + options = { + "--built": tmp_path / "out", + "--explanations": tmp_path / "e.csv", + "--actions": tmp_path / "a.json", + "--grounding": tmp_path / "g.csv", + } + argv = ["--stage-dir", str(tmp_path / "stage"), "--step", "install-references"] + for name, value in options.items(): + if name != missing: + argv += [name, str(value)] + with pytest.raises(SystemExit): + driver.parse_args(argv) + assert f"install-references requires {missing}" in capsys.readouterr().err + with pytest.raises(SystemExit): + driver.parse_args([*argv, missing, str(options[missing]), "--early"]) + + +def test_main_runs_install_references_on_a_resumable_stage(workspace, monkeypatch): + calls = [] + monkeypatch.setattr(driver, "install_references", lambda args: calls.append(args)) + stage = workspace / "results/local/stage" + _resumable(stage, None) + sources = {} + for name in ("out", "e.csv", "a.json", "g.csv"): + sources[name] = workspace / "results/local/build" / name + argv = ["--stage-dir", str(stage), "--step", "install-references"] + argv += ["--built", str(sources["out"]), "--explanations", str(sources["e.csv"])] + argv += ["--actions", str(sources["a.json"]), "--grounding", str(sources["g.csv"])] + driver.main(argv) + assert [args.built for args in calls] == [sources["out"].resolve()] + + +@pytest.mark.parametrize("step", ["triage", "export", "adjudicate-exclusions"]) +def test_later_steps_take_the_releases_record_from_the_installed_build( + workspace, monkeypatch, step +): + """With references_installed, main holds exclusions_installed to the + re-gated build's record, not to build_release_exclusions'.""" + built = SimpleNamespace(sha256={EXCLUSIONS_NAME: "b" * 64}) + monkeypatch.setattr(driver, "verify_installed_references", lambda *a: built) + monkeypatch.setattr( + driver, "release_exclusions_sha256", lambda: pytest.fail("fallback read") + ) + called = step.replace("-", "_") + calls = [] + for name in ("triage", "adjudicate_exclusions"): + monkeypatch.setattr(driver, name, lambda *a, n=name: calls.append(n)) + monkeypatch.setattr(driver, "export", lambda *a: calls.append("export")) + monkeypatch.setattr(driver, "resolve_live_base", lambda args: {}) + stage = workspace / "results/local/stage" + _resumable(stage, {"sha256": "b" * 64}) + receipt = json.loads((stage / "stage.json").read_text()) + receipt["references_installed"] = {"engine_version": MOCK_ENGINE} + (stage / "stage.json").write_text(json.dumps(receipt)) + driver.main(["--stage-dir", str(stage), "--step", step]) + assert calls[-1] == called + receipt["exclusions_installed"] = {"sha256": release_sha()} + (stage / "stage.json").write_text(json.dumps(receipt)) + with pytest.raises(SystemExit, match="run --step install-exclusions"): + driver.main(["--stage-dir", str(stage), "--step", step]) + + +# --- Adjudications with an upgrade installed ----------------------------------- + +# MOCK: the release 20261010 records the default `upgraded` build regenerates. +UPGRADED_REGENERATED = {**MOCK_REGENERATED, WI_042: 0.0} +UPGRADED_CASES = frozenset(driver.output_case(k) for k in UPGRADED_REGENERATED) + + +@pytest.fixture +def upgraded(tmp_path, monkeypatch, upgrade_spec): + """MOCK: a real-base build that regenerates four release 20261010 + engine-defect records and WI 042 and adds no exclusion, installed (as far + as the adjudication gates see) on a stage whose cases hold the verdicts + the decisions need.""" + build = real_build(tmp_path / "build", regenerated=UPGRADED_REGENERATED, added={}) + upgrade = load(build) + stage = tmp_path / "stage" + cases = _all_verdicts( + stage / "audit" / "cases", base_adjudication_record()["adjudications"] + ) + (stage / "stage.json").write_text(json.dumps({"files": {}})) + monkeypatch.setattr(driver, "stage_upgrade", lambda stage_dir: upgrade) + return SimpleNamespace(upgrade=upgrade, stage=stage, cases=cases, build=build) + + +def test_adjudicate_exclusions_drops_the_regenerated_decisions(upgraded): + """The five regenerated records' release 20261010 decisions are dropped + and the drops recorded; nothing is ruled or added, so the date conventions + stay release 20261010's. The record excludes exactly the build's 53 + records, and triage's gate accepts it.""" + from policybench.adjudications import excluded_case_keys, parse_adjudications + from policybench.reference_exclusions import exclusion_keys + + path = _write_stage_record(upgraded.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + written = json.loads(path.read_text()) + assert path.read_text() == driver.record_text(written) + entries = parse_adjudications(written, path) + cases = {driver.case_id(entry) for entry in entries} + assert len(entries) == 70 - 5 + assert not UPGRADED_CASES & cases + assert written["date_conventions"] == ( + base_adjudication_record()["date_conventions"] + ) + built = json.loads(upgraded.upgrade.exclusions_text)["exclusions"] + assert excluded_case_keys(entries) == exclusion_keys(built) + assert len(built) == 53 + receipt = json.loads((upgraded.stage / "stage.json").read_text()) + drops = receipt["adjudications_dropped"] + assert sorted(drop["case_id"] for drop in drops) == sorted(UPGRADED_CASES) + base = {driver.case_id(e): e for e in base_adjudication_record()["adjudications"]} + assert all(drop["entry"] == base[drop["case_id"]] for drop in drops) + assert all("#178" in drop["reason"] for drop in drops) + driver.verify_recorded_drops(upgraded.stage, upgraded.upgrade) + with mock.patch.object( + driver, "base_adjudication_record", base_adjudication_record + ): + staged = driver.stage_adjudications( + path, + UPGRADED_CASES, + [], + upgraded.cases, + regenerated=upgraded.upgrade.regenerated_ruled, + dropped=upgraded.upgrade.dropped_cases, + ) + assert len(staged) == 65 + + +def test_adjudicate_exclusions_refuses_a_second_run(upgraded): + """The first run dropped the regenerated decisions; a second finds no + decision to drop for them and writes nothing.""" + path = _write_stage_record(upgraded.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + text = path.read_text() + with pytest.raises(SystemExit, match="one decision for each regenerated"): + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + assert path.read_text() == text + + +# MOCK: the decisions on the two county outputs the mock build newly excludes. +MOCK_UPGRADE_ITEMS = [ + { + "scenario_id": key[0], + "variable": key[1], + "adjudicated_failure_source": "prompt_ambiguity", + "reference_verdict": "unlisted_input", + "adjudicated_failure_subtype": "state_local_rule", + "reasoning": "MOCK: the reference depends on the household's county.", + } + for key in sorted(MOCK_NEW) +] +MOCK_NEW_CASES = frozenset(driver.output_case(key) for key in MOCK_NEW) + + +@pytest.fixture +def upgraded_added(tmp_path, monkeypatch, upgrade_spec): + """MOCK: the default real-base build (four 20261010 engine-defect records + regenerated, two county outputs newly excluded), with the spec deciding + the county outputs and a bound verdict on each of their cases.""" + monkeypatch.setitem( + upgrade_spec, "upgrade_adjudications", copy.deepcopy(MOCK_UPGRADE_ITEMS) + ) + monkeypatch.setitem(upgrade_spec, "upgrade_adjudications_written_on", MOCK_DATE) + build = real_build(tmp_path / "build") + upgrade = load(build) + stage = tmp_path / "stage" + cases = _all_verdicts( + stage / "audit" / "cases", base_adjudication_record()["adjudications"] + ) + for case in sorted(MOCK_NEW_CASES): + directory = cases / case + directory.mkdir(parents=True, exist_ok=True) + (directory / "prompt.md").write_text(f"Classify {case}.\n") + write_verdict( + directory, _verdict([NEW]), judged_at_utc=f"{MOCK_DATE}T19:00:00+00:00" + ) + (stage / "stage.json").write_text(json.dumps({"files": {}})) + monkeypatch.setattr(driver, "stage_upgrade", lambda stage_dir: upgrade) + return SimpleNamespace(upgrade=upgrade, stage=stage, cases=cases, spec=upgrade_spec) + + +REOPENED_ADDED = MOCK_REGENERATED_CASES | MOCK_NEW_CASES + + +def test_adjudicate_exclusions_decides_the_outputs_the_upgrade_excludes( + upgraded_added, +): + """The two county outputs get new entries built from the spec's items, + their build records and their bound verdicts, dated by the records; the + date conventions name the upgrade's wave after 2026-10-10; the record + excludes exactly the build's 56 records, and triage's gate accepts it.""" + from policybench.adjudications import excluded_case_keys, parse_adjudications + from policybench.reference_exclusions import exclusion_keys + + path = _write_stage_record(upgraded_added.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded_added.stage)) + written = json.loads(path.read_text()) + entries = parse_adjudications(written, path) + by_case = {driver.case_id(entry): entry for entry in entries} + for item in MOCK_UPGRADE_ITEMS: + entry = by_case[driver.output_case(driver.spec_key(item))] + assert entry["excluded_from_scoring"] is True + assert entry["adjudicated_on"] == MOCK_DATE + assert entry["reference_verdict"] == "unlisted_input" + assert entry["reference_basis"] == "county_fips" + assert entry["reasoning"] == item["reasoning"] + assert entry["judged_on_utc"] == MOCK_DATE + assert [driver.case_id(e) for e in entries[-2:]] == sorted(MOCK_NEW_CASES) + assert len(entries) == 70 - 4 + 2 + conventions = written["date_conventions"] + assert f"2026-10-06, 2026-10-10 or {MOCK_DATE})." in conventions + assert ( + f"The {MOCK_DATE} wave's decisions exclude the outputs the {MOCK_DATE} " + "engine upgrade of the references moved onto an input the prompt does not " + f"state, and were written on {MOCK_DATE} UTC." in conventions + ) + built = json.loads(upgraded_added.upgrade.exclusions_text)["exclusions"] + assert len(built) == 56 + assert excluded_case_keys(entries) == exclusion_keys(built) + added = driver.upgrade_decisions(upgraded_added.upgrade) + with mock.patch.object( + driver, "base_adjudication_record", base_adjudication_record + ): + staged = driver.stage_adjudications( + path, + REOPENED_ADDED, + [], + upgraded_added.cases, + regenerated=upgraded_added.upgrade.regenerated_ruled, + dropped=upgraded_added.upgrade.dropped_cases, + added=added, + ) + assert len(staged) == len(entries) + # Without the upgrade's decisions the gate refuses the record. + with pytest.raises(SystemExit): + driver.stage_adjudications( + path, + REOPENED_ADDED, + [], + upgraded_added.cases, + regenerated=upgraded_added.upgrade.regenerated_ruled, + dropped=upgraded_added.upgrade.dropped_cases, + ) + + +@pytest.mark.parametrize( + "edit, message", + [ + (lambda entry: entry.update(reasoning="MOCK: other words."), "upgrade's entry"), + (lambda entry: entry.update(adjudicated_on="2026-10-10"), "upgrade's entry"), + (lambda entry: entry.update(judge_reference_suspect=True), "upgrade's"), + (None, "no staged decision"), + ], +) +def test_the_gate_holds_the_upgrades_decisions_to_their_items( + upgraded_added, edit, message +): + from policybench.adjudications import parse_adjudications + + path = _write_stage_record(upgraded_added.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded_added.stage)) + record = json.loads(path.read_text()) + case = sorted(MOCK_NEW_CASES)[0] + index = next( + i for i, e in enumerate(record["adjudications"]) if driver.case_id(e) == case + ) + if edit is None: + del record["adjudications"][index] + else: + edit(record["adjudications"][index]) + entries = parse_adjudications(record, path) + with pytest.raises(SystemExit, match=message): + driver.verify_adjudication_changes( + driver.base_adjudications(), + entries, + REOPENED_ADDED, + [], + upgraded_added.cases, + regenerated=upgraded_added.upgrade.regenerated_ruled, + dropped=upgraded_added.upgrade.dropped_cases, + added=driver.upgrade_decisions(upgraded_added.upgrade), + ) + + +def test_the_spec_must_decide_exactly_the_upgrades_new_exclusions(upgraded_added): + upgrade = upgraded_added.upgrade + spec = copy.deepcopy(upgraded_added.spec) + assert [driver.spec_key(i) for i, _ in driver.upgrade_decisions(upgrade, spec)] == ( + sorted(MOCK_NEW) + ) + for items in (MOCK_UPGRADE_ITEMS[:1], MOCK_UPGRADE_ITEMS * 2, []): + spec["upgrade_adjudications"] = copy.deepcopy(items) + with pytest.raises(SystemExit, match="must decide exactly"): + driver.upgrade_decisions(upgrade, spec) + # Without an upgrade the items are not read. + assert driver.upgrade_decisions(None, spec) == [] + + +# MOCK: a triage decision affirming WI 042's regenerated reference, whose +# release 20261010 decision the upgrade drops; dated by the upgrade's wave. +MOCK_TRIAGE = { + "scenario_id": WI_042[0], + "variable": WI_042[1], + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "taxable_income_or_deductions", + "adjudicated_on": MOCK_DATE, + "reference_verdict": "affirmed", + "reference_basis": "MOCK basis", + "reasoning": "MOCK: the judge's reference hypothesis fails.", +} + + +def test_a_triage_decision_affirms_a_regenerated_reference( + upgraded, upgrade_spec, monkeypatch +): + """WI 042's 20261010 decision is dropped, and the triage decision takes its + case: a new entry from the item and the case's bound verdict, which the + triage gate accepts and holds exact. The upgrade's wave is named, as one + whose decisions affirm a reference.""" + from policybench.adjudications import parse_adjudications + + monkeypatch.setitem(upgrade_spec, "triage_adjudications", [dict(MOCK_TRIAGE)]) + monkeypatch.setitem(upgrade_spec, "upgrade_adjudications_written_on", MOCK_DATE) + spec = driver.load_spec() + assert spec["triage_adjudications"] == [MOCK_TRIAGE] + path = _write_stage_record(upgraded.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + record = json.loads(path.read_text()) + assert ( + f"The {MOCK_DATE} wave's decisions affirm references the {MOCK_DATE} " + "engine upgrade of the references changed" + ) in record["date_conventions"] + entries = parse_adjudications(record, path) + entry = next(e for e in entries if driver.case_id(e) == WI_042_CASE) + assert list(entry) == list(driver.TRIAGE_FIELDS) + assert "excluded_from_scoring" not in entry + assert entry["reference_verdict"] == "affirmed" + assert entry["reasoning"] == MOCK_TRIAGE["reasoning"] + triage = driver.triage_items(upgraded.upgrade) + gate = dict( + regenerated=upgraded.upgrade.regenerated_ruled, + dropped=upgraded.upgrade.dropped_cases, + triage=triage, + ) + base = driver.base_adjudications() + assert ( + driver.verify_adjudication_changes( + base, entries, UPGRADED_CASES, [], upgraded.cases, **gate + ) + == 0 + ) + # Without the triage item the gate refuses the entry as a kept decision. + with pytest.raises(SystemExit, match="keep the decisions"): + driver.verify_adjudication_changes( + base, + entries, + UPGRADED_CASES, + [], + upgraded.cases, + regenerated=upgraded.upgrade.regenerated_ruled, + dropped=upgraded.upgrade.dropped_cases, + ) + edited = copy.deepcopy(entries) + next(e for e in edited if driver.case_id(e) == WI_042_CASE)["reasoning"] = "x" + with pytest.raises(SystemExit, match="triage decision's entry"): + driver.verify_adjudication_changes( + base, edited, UPGRADED_CASES, [], upgraded.cases, **gate + ) + with pytest.raises(SystemExit, match="did not re-open"): + driver.verify_adjudication_changes( + base, entries, UPGRADED_CASES - {WI_042_CASE}, [], upgraded.cases, **gate + ) + + +@pytest.mark.parametrize( + "change, message", + [ + ({"reference_verdict": "excluded"}, "may only affirm"), + ({"adjudicated_on": "2026-10-10"}, "dated by the engine upgrade's wave"), + ({"extra": 1}, "must be items with"), + ], +) +def test_a_triage_decision_must_affirm_on_the_upgrades_wave( + upgraded, upgrade_spec, monkeypatch, change, message +): + item = {**MOCK_TRIAGE, **change} + monkeypatch.setitem(upgrade_spec, "triage_adjudications", [item]) + monkeypatch.setitem(upgrade_spec, "upgrade_adjudications_written_on", MOCK_DATE) + path = _write_stage_record(upgraded.stage, base_adjudication_record()) + text = path.read_text() + with pytest.raises(SystemExit, match=message): + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + assert path.read_text() == text + + +def test_triage_decisions_are_read_only_with_an_upgrade(): + spec = {"triage_adjudications": [MOCK_TRIAGE]} + assert driver.triage_items(None, spec) == [] + upgrade = SimpleNamespace(revision={"date": MOCK_DATE}) + assert driver.triage_items(upgrade, spec) == [MOCK_TRIAGE] + + +def test_the_upgrades_wave_needs_its_written_day(upgraded_added, monkeypatch): + monkeypatch.delitem(upgraded_added.spec, "upgrade_adjudications_written_on") + path = _write_stage_record(upgraded_added.stage, base_adjudication_record()) + text = path.read_text() + with pytest.raises(SystemExit, match="upgrade_adjudications_written_on"): + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded_added.stage)) + assert path.read_text() == text + + +@pytest.mark.parametrize( + "added, triage, expected", + [ + ([], [], None), + ([({}, {"decided_on": "d"})], [], ("d", True, False)), + ([], [{"adjudicated_on": "d"}], ("d", False, True)), + ([({}, {"decided_on": "d"})], [{"adjudicated_on": "d"}], ("d", True, True)), + ], +) +def test_the_decision_wave_names_its_day_and_kinds(added, triage, expected): + wave = driver.decision_wave(added, triage) + assert (None if wave is None else (wave.day, wave.excludes, wave.affirms)) == ( + expected + ) + with pytest.raises(SystemExit, match="several days"): + driver.decision_wave([({}, {"decided_on": "d"})], [{"adjudicated_on": "e"}]) + + +@pytest.fixture +def decided(upgraded): + """The staged record adjudicate-exclusions writes on the upgraded stage.""" + from policybench.adjudications import parse_adjudications + + path = _write_stage_record(upgraded.stage, base_adjudication_record()) + driver.adjudicate_exclusions(SimpleNamespace(stage_dir=upgraded.stage)) + record = json.loads(path.read_text()) + return SimpleNamespace( + **vars(upgraded), + path=path, + record=record, + entries=parse_adjudications(record, path), + ) + + +def _gate(decided, entries, **overrides): + options = { + "regenerated": decided.upgrade.regenerated_ruled, + "dropped": decided.upgrade.dropped_cases, + } + options.update(overrides) + return driver.verify_adjudication_changes( + driver.base_adjudications(), + entries, + UPGRADED_CASES, + [], + decided.cases, + **options, + ) + + +def test_the_adjudication_gate_accepts_exactly_the_upgrades_drops(decided): + assert _gate(decided, decided.entries) == 0 + base = {driver.case_id(e): e for e in base_adjudication_record()["adjudications"]} + # WI 042's decision kept: refused. + with pytest.raises(SystemExit, match="keep the decisions of records the engine"): + _gate(decided, [*decided.entries, base[WI_042_CASE]]) + # Another decision dropped: refused. + other = [ + e + for e in decided.entries + if driver.case_id(e) != driver.case_id(decided.record["adjudications"][0]) + ] + with pytest.raises(SystemExit, match="drop recorded decisions"): + _gate(decided, other) + # Without the upgrade's view the same record drops five decisions: refused. + with pytest.raises(SystemExit, match="drop recorded decisions"): + _gate(decided, decided.entries, regenerated=frozenset(), dropped=frozenset()) + # A drop of a decision release 20261010 lacks: refused. + with pytest.raises(SystemExit, match="drop decisions release 20261010 lacks"): + _gate( + decided, + decided.entries, + dropped=decided.upgrade.dropped_cases | {"us__x__y"}, + ) + + +def test_triage_requires_the_recorded_drops(decided): + receipt_path = decided.stage / "stage.json" + receipt = json.loads(receipt_path.read_text()) + driver.verify_recorded_drops(decided.stage, decided.upgrade) + for edit, problem in ( + (lambda r: r.pop("adjudications_dropped"), "records no dropped"), + (lambda r: r.update(adjudications_dropped=[]), "are not the regenerated"), + ( + lambda r: r["adjudications_dropped"][0]["entry"].update(reasoning="X."), + "not release 20261010's", + ), + ): + edited = copy.deepcopy(receipt) + edit(edited) + receipt_path.write_text(json.dumps(edited)) + with pytest.raises(SystemExit, match=problem): + driver.verify_recorded_drops(decided.stage, decided.upgrade) + + +@settings( + max_examples=40, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_the_excluded_set_equality_holds_for_any_dropped_decisions(unruled_spec, data): + """For any set of release 20261010's excluding decisions dropped as + regenerated: the gate accepts the record without them exactly when the + drop set is that set, and the record then excludes 20261010's exclusions + less the regenerated outputs, as the build's record does.""" + from policybench.adjudications import excluded_case_keys + + base = driver.base_adjudications() + excluding = sorted( + driver.case_id(e) for e in base if e.get("excluded_from_scoring") + ) + dropped = frozenset(data.draw(st.sets(st.sampled_from(excluding), max_size=6))) + staged = [e for e in base if driver.case_id(e) not in dropped] + with tempfile.TemporaryDirectory() as scratch: + cases = Path(scratch) + assert ( + driver.verify_adjudication_changes( + base, staged, frozenset(), [], cases, dropped=dropped + ) + == 0 + ) + expected = excluded_case_keys(base) - { + tuple(case.split("__", 2)[1:]) for case in dropped + } + assert excluded_case_keys(staged) == expected + other = data.draw(st.sampled_from(excluding)) + wrong = dropped ^ {other} + with pytest.raises(SystemExit): + driver.verify_adjudication_changes( + base, staged, frozenset(), [], cases, dropped=wrong + ) + + +# --- Export with an upgrade installed ------------------------------------------ + + +@pytest.fixture +def upgrade_bundle(tmp_path, upgrade_spec): + """MOCK: a bundle holding a real-base build's references, the export + inputs the scope export copies, and the build gated.""" + build = real_build(tmp_path / "build") + upgrade = load(build) + bundle = tmp_path / "stage" / "publish" / driver.RUN_NAME + base_references(bundle / "us") + for name in driver.UPGRADED_FILES: + (bundle / "us" / name).write_bytes((build.built / name).read_bytes()) + (bundle / "us" / "predictions.csv").write_text("model,scenario_id,variable\n") + (bundle / "annotations").mkdir() + for name in ("us_audit_row_annotations.csv", "us_case_notes.csv"): + (bundle / "annotations" / name).write_text("country\n") + (bundle / "annotations" / driver.EXPLANATIONS_NAME).write_bytes( + build.explanations.read_bytes() + ) + return SimpleNamespace(build=build, upgrade=upgrade, bundle=bundle) + + +def test_the_upgrade_scope_check_puts_back_the_base_references_and_record( + scope, upgrade_bundle +): + """(a) The scope export reads release 20261010's five reference files and + exclusion record, never the build's; (b) every model is scored on 1,984 + less the build's 56 records.""" + incumbents = _incumbents() + stats = _exported(incumbents) + for row in stats: + row["n"] = 1928 + payload = scope.build( + stats, + _scoped(incumbents), + incumbents, + bundle=upgrade_bundle.bundle, + upgrade=upgrade_bundle.upgrade, + ) + assert {row["n"] for row in payload["countries"]["us"]["modelStats"]} == {1928} + # The scope export relaxes only the annotation requirement; the release's + # own payload keeps it. + assert scope.scope_gates == [False] and scope.gates[-1] is True + (references,) = scope.references + assert references == { + name: base_blob(SNAPSHOT_PATH / name) for name in driver.REFERENCE_FILES + } + (_, _, _, record) = scope.calls[-1] + assert sha(record) == driver.BASE_REFERENCE_SHA256[EXCLUSIONS_NAME] + # The scratch copy is gone; the bundle keeps the build's files. + assert (upgrade_bundle.bundle / "us" / CSV_NAME).read_bytes() == ( + upgrade_bundle.build.built / CSV_NAME + ).read_bytes() + assert [p.name for p in upgrade_bundle.bundle.parent.iterdir()] == [driver.RUN_NAME] + + +@settings( + max_examples=40, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + counts=st.lists( + st.sampled_from([1928, 1928, 1926, 1920, 1927]), min_size=48, max_size=48 + ) +) +def test_with_an_upgrade_every_model_is_scored_on_1984_less_its_records( + scope, upgrade_bundle, counts +): + incumbents = _incumbents() + stats = _exported(incumbents) + for row, n in zip(stats, counts): + row["n"] = n + run = functools.partial( + scope.build, + stats, + _scoped(incumbents), + incumbents, + bundle=upgrade_bundle.bundle, + upgrade=upgrade_bundle.upgrade, + ) + if set(counts) == {1928}: + run() + else: + with pytest.raises(SystemExit, match="must be scored on 1928 outputs"): + run() + + +def test_the_upgrade_scope_check_refuses_incumbent_drift(scope, upgrade_bundle): + incumbents = _incumbents() + stats = _exported(incumbents) + for row in stats: + row["n"] = 1928 + scoped = _scoped(incumbents) + scoped[3]["exact"] += 0.01 + with pytest.raises( + SystemExit, + match=r"references and exclusion record, incumbent modelStats drift: " + r"\['incumbent-03'\]", + ): + scope.build( + stats, + scoped, + incumbents, + bundle=upgrade_bundle.bundle, + upgrade=upgrade_bundle.upgrade, + ) + + +def test_export_holds_the_staged_references_to_the_installed_build( + exporting, monkeypatch, upgrade_spec, tmp_path +): + """Export with an upgrade: the staged references must be the build's, the + committed ones release 20261010's until the freeze.""" + stage, bundle, run, _ = exporting + _evidence(stage, bundle) + build = real_build(tmp_path / "build") + upgrade = load(build) + monkeypatch.setattr(driver, "stage_upgrade", lambda stage_dir: upgrade) + # The annotation CSVs the scope export copies with the references. + for name in ("us_audit_row_annotations.csv", "us_case_notes.csv"): + (bundle / "annotations" / name).write_text("country\n") + (bundle / "annotations" / driver.EXPLANATIONS_NAME).write_bytes( + build.explanations.read_bytes() + ) + incumbents = _incumbents() + stats = _exported(incumbents) + for row in stats: + row["n"] = 1928 + with pytest.raises(SystemExit, match="it is not the installed engine upgrade's"): + run(stats, incumbents) + # install-references writes the build's files and rebinds them. + receipt = json.loads((stage / "stage.json").read_text()) + for name in driver.UPGRADED_FILES: + (bundle / "us" / name).write_bytes((build.built / name).read_bytes()) + receipt["files"][f"publish/{driver.RUN_NAME}/us/{name}"] = sha( + build.built / name + ) + (stage / "stage.json").write_text(json.dumps(receipt)) + run(stats, incumbents) + assert (stage / "release-ready.json").is_file() + + +def test_triage_holds_the_record_to_the_installed_upgrade(triage_stage, monkeypatch): + """With an upgrade installed, triage checks the recorded drops and gives + the adjudication gate the upgrade's regenerated outputs and drops.""" + stage, _, run = triage_stage + upgrade = SimpleNamespace( + regenerated_ruled=frozenset(MOCK_REGENERATED), + dropped_cases=frozenset({WI_042_CASE}), + added=frozenset(), + ) + monkeypatch.setattr(driver, "stage_upgrade", lambda stage_dir: upgrade) + # MOCK: the upgrade excludes nothing new and triages nothing, so it decides + # nothing new. + monkeypatch.setattr(driver, "upgrade_decisions", lambda u: []) + monkeypatch.setattr(driver, "triage_items", lambda u: []) + checked, given = [], [] + monkeypatch.setattr( + driver, "verify_recorded_drops", lambda *a: checked.append(a[1]) + ) + + def gate(*args, **kwargs): + given.append(kwargs) + raise SystemExit("gate reached") + + monkeypatch.setattr(driver, "stage_adjudications", gate) + with pytest.raises(SystemExit, match="gate reached"): + run() + assert checked == [upgrade] + assert given == [ + { + "regenerated": frozenset(MOCK_REGENERATED), + "dropped": frozenset({WI_042_CASE}), + "added": [], + "triage": [], + } + ] + + +def test_install_references_needs_the_seed_binding(upgrade_stage): + s = upgrade_stage + receipt = json.loads((s.stage / "stage.json").read_text()) + del receipt["seed"] + (s.stage / "stage.json").write_text(json.dumps(receipt)) + before = stage_files(s.stage) + with pytest.raises(SystemExit, match="does not bind the audit seed"): + s.install(s.build(S1_MOVE)) + assert stage_files(s.stage) == before From 0fb944a1b728ce611ec76ccbbe8ac39feda159c1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 20:04:08 -0400 Subject: [PATCH 06/28] Test the 2026-10-12 audit's sweep timing script, as the 2026-10-09 one is tested Co-Authored-By: Claude Opus 5.5 --- tests/test_sweep_timing_1012.py | 870 ++++++++++++++++++++++++++++++++ 1 file changed, 870 insertions(+) create mode 100644 tests/test_sweep_timing_1012.py diff --git a/tests/test_sweep_timing_1012.py b/tests/test_sweep_timing_1012.py new file mode 100644 index 00000000..84c8ea5c --- /dev/null +++ b/tests/test_sweep_timing_1012.py @@ -0,0 +1,870 @@ +"""reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py: the +2026-10-12 engine move's timing record and publication check, on MOCK PyPI +data. A copy of tests/test_sweep_timing.py, which tests the 2026-10-09 audit's +script, pointed at this audit's.""" + +import importlib.util +import json +import math +import time +from pathlib import Path + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +REPO = Path(__file__).resolve().parents[1] +_spec = importlib.util.spec_from_file_location( + "sweep_timing_1012", + REPO / "reference_audit/2026-10-12-engine-upgrade/scripts/sweep_timing.py", +) +timing = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(timing) + +# Linux's stat keeps no birth time, and the record's times are birth times. +BIRTH_TIMES = hasattr(Path(__file__).stat(), "st_birthtime") +REAL_BIRTH_TIME = timing.birth_time + + +@pytest.fixture(autouse=True) +def _birth_times(monkeypatch): + """MOCK, on a platform without birth times (Linux CI) only: date each file + by its modification time. These tests create every file they date once and + never rewrite it, so the two times agree; on macOS the real birth times + are read.""" + if not BIRTH_TIMES: + monkeypatch.setattr(timing, "birth_time", lambda path: path.stat().st_mtime) + + +def test_birth_time_refuses_a_platform_without_birth_times(tmp_path): + """Where stat keeps no birth time, the record refuses rather than date the + install or an output by a modification time.""" + path = tmp_path / "computed.csv" + path.write_text("x\n") + + class NoBirth: + def stat(self): + return type("Stat", (), {"st_mtime": 0.0})() + + def __str__(self): + return str(path) + + with pytest.raises(timing.Refusal, match="keeps no birth time"): + REAL_BIRTH_TIME(NoBirth()) + if BIRTH_TIMES: + assert REAL_BIRTH_TIME(path) == path.stat().st_birthtime + + +def _wheel(at, *, yanked=False, kind="bdist_wheel"): + return {"packagetype": kind, "upload_time_iso_8601": at, "yanked": yanked} + + +MOCK_PYPI = { + "releases": { + "2.38.4": [ + _wheel("2026-10-09T20:50:00.1Z"), + _wheel("2026-10-09T20:49:00Z", kind="sdist"), + ], + "2.39.0": [_wheel("2026-10-10T01:00:00.5Z")], + "2.39.1": [_wheel("2026-10-10T05:00:00Z", yanked=True)], + "2.40.0rc1": [_wheel("2026-10-10T06:00:00Z")], + "2.9.0": [_wheel("2026-08-01T00:00:00Z")], + "2.10.0": [], + } +} + + +def test_wheel_uploads_keep_released_wheels_only(): + """MOCK: sdists, yanked wheels, pre-releases and empty releases drop out; + versions order numerically, so 2.10 > 2.9.""" + uploads = timing.wheel_uploads(MOCK_PYPI) + assert uploads == { + "2.38.4": "2026-10-09T20:50:00.1Z", + "2.39.0": "2026-10-10T01:00:00.5Z", + "2.9.0": "2026-08-01T00:00:00Z", + } + assert timing.newest(uploads) == "2.39.0" + assert list(timing.from_engine(uploads, "2.38.4")) == ["2.38.4", "2.39.0"] + + +@pytest.mark.parametrize( + "moment, newer", + [ + ("2026-10-10T00:59:59Z", []), + ("2026-10-10T01:00:00Z", []), + ("2026-10-10T01:00:01Z", ["2.39.0"]), + ], +) +def test_a_newer_release_before_the_first_output_is_found(moment, newer): + uploads = timing.wheel_uploads(MOCK_PYPI) + assert timing.newer_before(uploads, "2.38.4", moment) == newer + + +def _computed(values: dict, path: Path) -> Path: + lines = ["fix,scenario_id,state,variable,frozen,recomputed,delta,moved"] + for (s, v), x in values.items(): + lines.append(f"latest_final,{s},TX,{v},0,{x!r},0,False") + path.write_text("\n".join(lines) + "\n") + return path + + +def test_compare_counts_the_outputs_that_differ(tmp_path): + a = _computed({("s1", "v"): 1.0, ("s2", "v"): 2.0}, tmp_path / "a.csv") + b = _computed({("s1", "v"): 1.0, ("s2", "v"): 2.5}, tmp_path / "b.csv") + rows, summary = timing.compare( + timing.read_computed(a), timing.read_computed(b), "2.38.4", "2.39.0" + ) + assert summary == { + "outputs": 2, + "same": 1, + "differ": ["s2|v"], + "scored_outputs": 2, + "scored_same": 1, + "scored_differ": ["s2|v"], + "excluded_differ": [], + "max_abs_delta": 0.5, + } + assert [r["same"] for r in rows] == [True, False] + # The same move on an output the release excludes touches no scored one. + _, excluded = timing.compare( + timing.read_computed(a), + timing.read_computed(b), + "2.38.4", + "2.39.0", + frozenset({("s2", "v")}), + ) + assert (excluded["scored_outputs"], excluded["scored_same"]) == (1, 1) + assert excluded["scored_differ"] == [] and excluded["excluded_differ"] == ["s2|v"] + with pytest.raises(timing.Refusal, match="do not cover"): + timing.compare( + timing.read_computed(a), + timing.read_computed(b), + "x", + "y", + frozenset({("s9", "v")}), + ) + + +def test_compare_refuses_other_outputs_or_non_finite_values(tmp_path): + a = timing.read_computed(_computed({("s1", "v"): 1.0}, tmp_path / "a.csv")) + b = timing.read_computed(_computed({("s2", "v"): 1.0}, tmp_path / "b.csv")) + with pytest.raises(timing.Refusal, match="different outputs"): + timing.compare(a, b, "x", "y") + nan = timing.read_computed(_computed({("s1", "v"): math.nan}, tmp_path / "n.csv")) + with pytest.raises(timing.Refusal, match="non-finite"): + timing.compare(a, nan, "x", "y") + + +@given( + st.dictionaries( + st.text("abc", min_size=1, max_size=3), st.floats(-1e9, 1e9), min_size=1 + ) +) +def test_a_sweep_compared_with_itself_is_all_the_same(values): + """Property: every output of a sweep is the same as itself.""" + computed = {(k, "v"): ("TX", x) for k, x in values.items()} + rows, summary = timing.compare(computed, computed, "a", "a") + assert summary["same"] == summary["outputs"] == len(values) + assert summary["differ"] == [] and summary["max_abs_delta"] == 0.0 + + +def _venv(root: Path, engine: str) -> Path: + (root / f"lib/python3.12/site-packages/policyengine_us-{engine}.dist-info").mkdir( + parents=True + ) + return root + + +def _pypi_around_now(offset_newer: float, offset_engine: float = -3600) -> dict: + """MOCK PyPI: the engine offset_engine seconds from now (an hour ago by + default), a newer release offset_newer seconds from now.""" + + def at(seconds): + return timing.utc(time.time() + seconds) + + return { + "releases": { + "2.38.4": [_wheel(at(offset_engine))], + "2.39.0": [_wheel(at(offset_newer))], + } + } + + +def _sweep_args(tmp_path: Path): + """MOCK sweep inputs, in a sweep's order: the engine is installed, then + the sweep writes its first output.""" + venv = _venv(tmp_path / "venv", "2.38.4") + first = tmp_path / "computed.csv" + first.write_text("x\n") + return type( + "Args", + (), + {"engine": "2.38.4", "venv": str(venv), "first_output": str(first)}, + ) + + +MOCK_INSTALL = {"version": "MOCK", "files_verified": 1, "editable": False} + + +@pytest.mark.parametrize("offset, ok", [(600, True), (-600, False)]) +def test_the_sweep_refuses_an_engine_that_was_not_the_newest( + tmp_path, monkeypatch, offset, ok +): + """MOCK: a release newer than the engine uploaded before the sweep's first + output means the engine was not the newest when the sweep began.""" + args = _sweep_args(tmp_path) + monkeypatch.setattr(timing, "engine_install", lambda venv, engine: MOCK_INSTALL) + monkeypatch.setattr(timing, "read_pypi", lambda url=None: _pypi_around_now(offset)) + if ok: + record = timing.sweep(args) + assert record["reference_sweep"]["engine"] == "2.38.4" + assert record["pypi"]["newest_at_read"] == "2.39.0" + assert ( + record["reference_sweep"]["first_output_at_utc"] + <= record["pypi"]["read_at_utc"] + ) + assert record["reference_sweep"]["first_output_sha256"] == ( + timing.sha256(Path(args.first_output)) + ) + else: + with pytest.raises(timing.Refusal, match="was not the newest"): + timing.sweep(args) + + +def test_the_sweep_refuses_an_engine_uploaded_after_its_output(tmp_path, monkeypatch): + """MOCK: an output older than its engine's wheel (an earlier rehearsal's + file, say) cannot date the sweep.""" + args = _sweep_args(tmp_path) + monkeypatch.setattr(timing, "engine_install", lambda venv, engine: MOCK_INSTALL) + monkeypatch.setattr( + timing, "read_pypi", lambda url=None: _pypi_around_now(7200, offset_engine=3600) + ) + with pytest.raises(timing.Refusal, match="must follow"): + timing.sweep(args) + + +_T = st.integers(0, 10).map(lambda n: f"2026-10-10T0{n // 10}:{n % 10}0:00Z") + + +@given(_T, _T, _T) +def test_a_sweep_follows_its_engines_upload_and_install(uploaded, installed, output): + """Property: the order check passes exactly when upload <= install <= output.""" + if uploaded <= installed <= output: + timing.sweep_order_problems(uploaded, installed, output, "2.38.4") + else: + with pytest.raises(timing.Refusal, match="must follow"): + timing.sweep_order_problems(uploaded, installed, output, "2.38.4") + + +def _write_build(root: Path, values: dict, excluded: set, engine: str = "2.38.4"): + """MOCK reference build and the committed snapshot it froze into: the + references are the computed values, every output scored but ``excluded``, + and the sidecar names the committed builder.""" + build, snapshot = root / "build", root / "snapshot" + build.mkdir(parents=True) + snapshot.mkdir(parents=True) + _computed(values, build / "computed.csv") + lines = ["scenario_id,variable,value,impact_weight"] + lines += [f"{s},{v},{x!r}," for (s, v), x in values.items()] + (build / "reference_outputs.csv").write_text("\n".join(lines) + "\n") + record = { + "exclusions": [{"scenario_id": s, "variable": v} for s, v in sorted(excluded)] + } + (build / "reference_exclusions.json").write_text(json.dumps(record)) + meta = { + "reference_csv_sha256": timing.sha256(build / "reference_outputs.csv"), + "revisions": [ + { + "kind": "engine_upgrade", + "engine_version": f"policyengine-us {engine}", + "provenance": {"builder_sha256": timing.sha256(timing.BUILDER)}, + } + ], + } + (build / "reference_outputs.csv.meta.json").write_text(json.dumps(meta)) + for name in ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + ): + (snapshot / name).write_bytes((build / name).read_bytes()) + return build, snapshot + + +VALUES = {("s1", "v"): 1.0, ("s2", "v"): 2.0, ("s3", "v"): 3.0} + + +def _check_setup(tmp_path, monkeypatch, pypi, excluded=frozenset({("s3", "v")})): + path = tmp_path / "sweep_timing.json" + path.write_text(json.dumps({"reference_sweep": {"engine": "2.38.4"}, "pypi": {}})) + monkeypatch.setattr(timing, "TIMING", path) + monkeypatch.setattr(timing, "VERIFICATION", tmp_path / "verification") + monkeypatch.setattr(timing, "read_pypi", lambda url=None: pypi) + build, snapshot = _write_build(tmp_path, VALUES, set(excluded)) + return build, snapshot + + +def _check_args(build, snapshot, check_venv=None, work_dir=None): + return type( + "Args", + (), + { + "build": str(build), + "snapshot": str(snapshot), + "check_venv": check_venv and str(check_venv), + "work_dir": work_dir and str(work_dir), + }, + ) + + +def _MOCK_sweep(values: dict): + """MOCK check sweep: writes ``values`` as the computed.csv a first pass on + the venv's engine would, in place of running the builder.""" + + def run(venv, engine, out_dir): + out_dir.mkdir(parents=True) + path = _computed(values, out_dir / "computed.csv") + return path, { + "builder_sha256": timing.sha256(timing.BUILDER), + "MOCK": True, + "engine_install": MOCK_INSTALL, + } + + return run + + +ONLY_ENGINE = {"releases": {"2.38.4": [_wheel("2026-10-09T20:50:00Z")]}} + + +def test_the_check_records_a_still_newest_engine(tmp_path, monkeypatch): + """MOCK: with the reference engine still the newest, the record says so and + pins the build; a newer release needs a venv holding it.""" + build, snapshot = _check_setup(tmp_path, monkeypatch, ONLY_ENGINE) + out = timing.check(_check_args(build, snapshot)) + check = out["publication_check"] + assert check["engine"] == "2.38.4" and "was still the newest" in check["result"] + assert check["scored_outputs"] == 2 + assert set(check["build_sha256"]) == {*timing.BUILD_FILES, "builder"} + monkeypatch.setattr(timing, "read_pypi", lambda url=None: MOCK_PYPI) + with pytest.raises(timing.Refusal, match="newer than the reference engine"): + timing.check(_check_args(build, snapshot)) + + +def test_the_check_holds_its_build_to_the_release(tmp_path, monkeypatch): + """MOCK: a build that is not the frozen release's is refused before PyPI + is read: another engine, a sidecar that does not pin its CSV, a missing or + other builder, a sidecar, CSV or record that is not the committed + snapshot's, a computed.csv that is not the scored references, and a + non-finite value anywhere.""" + build, snapshot = _check_setup(tmp_path, monkeypatch, ONLY_ENGINE) + meta_path = build / "reference_outputs.csv.meta.json" + meta = json.loads(meta_path.read_text()) + revision = meta["revisions"][0] + + def refused(match): + with pytest.raises(timing.Refusal, match=match): + timing.check(_check_args(build, snapshot)) + + def sidecar(**changes): + meta_path.write_text(json.dumps({**meta, **changes})) + + sidecar(revisions=[{**revision, "engine_version": "x"}]) + refused("not an engine upgrade to 2.38.4") + sidecar(reference_csv_sha256="0" * 64) + refused("does not pin its reference_outputs.csv") + for provenance in ({"builder_sha256": "f" * 64}, {}): + sidecar(revisions=[{**revision, "provenance": provenance}]) + refused("not the committed one") + sidecar(MOCK_extra=1) + refused("reference_outputs.csv.meta.json is not the committed snapshot's") + sidecar() + record = snapshot / "reference_exclusions.json" + original = record.read_bytes() + record.write_text(json.dumps({"exclusions": []})) + refused("reference_exclusions.json is not the committed snapshot's") + record.write_bytes(original) + _computed({**VALUES, ("s1", "v"): 1.5}, build / "computed.csv") + refused("not its scored references") + # Non-finite values are refused, scored or excluded, before any + # comparison: the still-newest branch never reaches compare. + for key in (("s1", "v"), ("s3", "v")): + _computed({**VALUES, key: math.nan}, build / "computed.csv") + refused("non-finite") + # A moved excluded output is the engine's value, not the reference's: fine. + _computed({**VALUES, ("s3", "v"): 9.0}, build / "computed.csv") + assert timing.check(_check_args(build, snapshot))["publication_check"] + + +@pytest.mark.parametrize( + "moved, publishes", + [({}, True), ({("s3", "v"): 3.5}, True), ({("s2", "v"): 2.5}, False)], +) +def test_the_check_stops_the_publish_when_a_scored_output_moves( + tmp_path, monkeypatch, moved, publishes +): + """MOCK: check runs the sweep on the newer release itself (here a MOCK + sweep); one that moves only excluded outputs passes, one that moves a + scored output writes the record and exits non-zero.""" + build, snapshot = _check_setup( + tmp_path, monkeypatch, _pypi_around_now(-1800, offset_engine=-7200) + ) + venv = _venv(tmp_path / "check-venv", "2.39.0") + monkeypatch.setattr(timing, "engine_install", lambda venv, engine: MOCK_INSTALL) + monkeypatch.setattr(timing, "run_check_sweep", _MOCK_sweep({**VALUES, **moved})) + monkeypatch.setattr(timing.tempfile, "mkdtemp", lambda **kw: str(tmp_path / "w")) + monkeypatch.setattr(timing, "SNAPSHOT", snapshot) + argv = ["check", "--build", str(build), "--check-venv", str(venv)] + if publishes: + timing.main(argv) + else: + with pytest.raises(timing.Refusal, match="do not publish"): + timing.main(argv) + check = json.loads(timing.TIMING.read_text())["publication_check"] + assert check["engine"] == "2.39.0" and check["scored_outputs"] == 2 + assert bool(check["scored_differ"]) is not publishes + assert check["check_sweep"]["MOCK"] is True + assert check["check_sweep"]["engine_install"] == MOCK_INSTALL + # A venv installed before the newer release's wheel was uploaded does not + # hold that release. + monkeypatch.setattr( + timing, + "read_pypi", + lambda url=None: _pypi_around_now(3600, offset_engine=-7200), + ) + with pytest.raises(timing.Refusal, match="must follow"): + timing.check(_check_args(build, snapshot, venv, tmp_path / "w2")) + + +def _fake_run_writing(receipt_problems=(), computed=True): + """MOCK subprocess.run for run_on_engine: writes the engine check's + receipt (and the sweep's computed.csv) where the real runner would.""" + calls = [] + + def run(command, env=None, **kw): + calls.append((command, env)) + Path(env["PB_ENGINE_RECEIPT"]).write_text( + json.dumps( + { + "version": env["PB_EXPECTED_ENGINE"], + "files_verified": 2, + "problems": list(receipt_problems), + } + ) + ) + if computed and "--out-dir" in command: + out = Path(command[command.index("--out-dir") + 1]) + out.mkdir(parents=True, exist_ok=True) # as the builder does + _computed(VALUES, out / "computed.csv") + return type("Ran", (), {"returncode": 1, "stdout": "", "stderr": "draft"})() + + return run, calls + + +def test_the_check_sweep_runs_the_builder_in_the_checked_process(tmp_path, monkeypatch): + """MOCK subprocess: one process, under the venv's interpreter with -P and a + fresh bytecode-cache prefix, checks the engine and then runs the builder + with empty actions naming the newer engine; a sweep that writes nothing, + or a failed check, is refused.""" + run, calls = _fake_run_writing() + monkeypatch.setattr(timing.subprocess, "run", run) + venv = tmp_path / "venv" + path, receipt = timing.run_check_sweep(venv, "2.39.0", tmp_path / "a") + assert path.is_file() and receipt["exit_code"] == 1 + assert receipt["engine_install"]["fresh_bytecode_cache"] is True + command, env = calls[0] + assert command[:4] == [str(venv / "bin/python"), "-P", "-c", timing.ENGINE_RUNNER] + assert command[4] == str(timing.BUILDER) and "--allow-draft" in command + assert env["PB_EXPECTED_ENGINE"] == "2.39.0" and env["PYTHONPYCACHEPREFIX"] + actions = json.loads((tmp_path / "a/actions.empty.json").read_text()) + assert actions["engine"] == "policyengine-us 2.39.0" and actions["draft"] is True + run, _ = _fake_run_writing(computed=False) + monkeypatch.setattr(timing.subprocess, "run", run) + with pytest.raises(timing.Refusal, match="wrote no computed.csv"): + timing.run_check_sweep(venv, "2.39.0", tmp_path / "b") + run, _ = _fake_run_writing(receipt_problems=["an editable install"]) + monkeypatch.setattr(timing.subprocess, "run", run) + with pytest.raises(timing.Refusal, match="an editable install"): + timing.run_check_sweep(venv, "2.39.0", tmp_path / "c") + monkeypatch.setattr( + timing.subprocess, + "run", + lambda *a, **k: type( + "Ran", (), {"returncode": 2, "stdout": "", "stderr": "x"} + )(), + ) + with pytest.raises(timing.Refusal, match="did not run"): + timing.engine_install(venv, "2.39.0") + + +def test_a_relative_check_venv_is_the_one_found_and_run(tmp_path, monkeypatch): + """MOCK: --check-venv given relative to a caller outside the repository is + resolved once, so discovery, the install check and the sweep all use it.""" + build, snapshot = _check_setup( + tmp_path, monkeypatch, _pypi_around_now(-1800, offset_engine=-7200) + ) + venv = _venv(tmp_path / "check-venv", "2.39.0").resolve() + seen = [] + + def install(found, engine): + seen.append(found) + return MOCK_INSTALL + + def sweep(found, engine, out_dir): + seen.append(found) + return _MOCK_sweep(VALUES)(found, engine, out_dir) + + monkeypatch.setattr(timing, "engine_install", install) + monkeypatch.setattr(timing, "run_check_sweep", sweep) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + timing.check(_check_args(build, snapshot, "../check-venv", tmp_path / "w")) + assert seen == [venv, venv] + + +def test_a_missing_scratch_parent_is_created(tmp_path, monkeypatch): + """MOCK: in a checkout without results/local, check creates it before + making the sweep's scratch directory.""" + build, snapshot = _check_setup( + tmp_path, monkeypatch, _pypi_around_now(-1800, offset_engine=-7200) + ) + root = tmp_path / "clean-checkout" + root.mkdir() + monkeypatch.setattr(timing, "ROOT", root) + monkeypatch.setattr(timing, "engine_install", lambda venv, engine: MOCK_INSTALL) + monkeypatch.setattr(timing, "run_check_sweep", _MOCK_sweep(VALUES)) + venv = _venv(tmp_path / "check-venv", "2.39.0") + timing.check(_check_args(build, snapshot, venv)) + assert (root / "results/local").is_dir() + assert any((root / "results/local").glob("check-sweep-2.39.0-*")) + + +def _fake_install( + root: Path, version: str, files: dict[str, str], algorithm: str = "sha256" +) -> Path: + """MOCK site-packages: a policyengine-us wheel's dist-info whose RECORD + pins ``files`` as written, with the given hash algorithm.""" + import base64 + import hashlib + + site = root / "site" + record = [] + for name, text in files.items(): + path = site / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + digest = base64.urlsafe_b64encode( + hashlib.new(algorithm, text.encode()).digest() + ) + record.append(f"{name},{algorithm}={digest.rstrip(b'=').decode()},{len(text)}") + info = site / f"policyengine_us-{version}.dist-info" + info.mkdir() + (info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: policyengine-us\nVersion: {version}\n" + ) + (info / "RECORD").write_text("\n".join(record) + "\n") + return site + + +def _runner( + tmp_path: Path, *paths: Path, builder=None, fresh_cache=True, expected="9.9.9" +): + """Run ENGINE_RUNNER as run_on_engine does (-P, a fresh cache prefix), with + ``paths`` first on the import path, as a shadowing tree or a .pth redirect + would put them. Returns the receipt and the process.""" + import os + import subprocess + import sys + import tempfile + + receipt = Path(tempfile.mkdtemp(dir=tmp_path)) / "receipt.json" + env = { + **{k: v for k, v in os.environ.items() if k in ("PATH", "HOME", "LANG")}, + "PYTHONPATH": os.pathsep.join(str(p) for p in paths), + "PYTHONDONTWRITEBYTECODE": "1", + "PB_EXPECTED_ENGINE": expected, + "PB_ENGINE_RECEIPT": str(receipt), + } + if fresh_cache: + env["PYTHONPYCACHEPREFIX"] = tempfile.mkdtemp(dir=tmp_path) + command = [sys.executable, "-P", "-c", timing.ENGINE_RUNNER] + if builder is not None: + command.append(str(builder)) + ran = subprocess.run(command, env=env, capture_output=True, text=True, cwd=tmp_path) + return json.loads(receipt.read_text()), ran + + +PACKAGE = {"policyengine_us/__init__.py": "X = 1\n", "policyengine_us/m.py": "Y = 2\n"} + + +def _printer(directory: Path) -> Path: + """A MOCK builder that prints the engine module's X.""" + directory.mkdir(parents=True, exist_ok=True) + script = directory / "builder.py" + script.write_text("import policyengine_us\nprint('X', policyengine_us.X)\n") + return script + + +@pytest.mark.parametrize("algorithm", ["sha256", "sha384", "sha512"]) +def test_a_clean_wheel_passes_the_runner_and_runs_its_builder(tmp_path, algorithm): + """MOCK install, real runner: a clean wheel, with any hash RECORD permits, + passes and the builder runs in the same process on its code.""" + site = _fake_install(tmp_path / "clean", "9.9.9", PACKAGE, algorithm) + found, ran = _runner(tmp_path, site, builder=_printer(tmp_path / "b")) + assert found["problems"] == [] and found["files_verified"] == 2 + assert ran.returncode == 0 and "X 1" in ran.stdout + + +def test_the_runner_refuses_what_is_imported_not_the_label(tmp_path): + """MOCK installs, real runner: a source tree shadowing the wheel under the + same version label, an editable install, a file changed after install, an + unrecorded module, an unrecorded native extension, a wrong version and a + run without a fresh bytecode cache each stop it before any builder runs.""" + import importlib.machinery + + site = _fake_install(tmp_path / "clean", "9.9.9", PACKAGE) + shadow = tmp_path / "shadow" + (shadow / "policyengine_us").mkdir(parents=True) + (shadow / "policyengine_us" / "__init__.py").write_text("X = 0\n") + + def problems(*paths, **kw): + found, ran = _runner(tmp_path, *paths, builder=_printer(tmp_path / "p"), **kw) + assert ran.returncode == 3 and "X " not in ran.stdout, ran.stdout + return " ".join(found["problems"]) + + assert "imports policyengine_us from" in problems(shadow, site) + editable = _fake_install(tmp_path / "editable", "9.9.9", PACKAGE) + (editable / "policyengine_us-9.9.9.dist-info" / "direct_url.json").write_text( + json.dumps({"url": "file:///src", "dir_info": {"editable": True}}) + ) + assert "an editable install" in problems(editable) + changed = _fake_install(tmp_path / "changed", "9.9.9", PACKAGE) + (changed / "policyengine_us" / "m.py").write_text("Y = 3\n") + assert "policyengine_us/m.py" in problems(changed) + extra = _fake_install(tmp_path / "extra", "9.9.9", PACKAGE) + (extra / "policyengine_us" / "extra.py").write_text("Z = 4\n") + assert "importable files the wheel lacks" in problems(extra) + native = _fake_install(tmp_path / "native", "9.9.9", PACKAGE) + suffix = importlib.machinery.EXTENSION_SUFFIXES[0] + (native / "policyengine_us" / f"m{suffix}").write_bytes(b"\0") + assert f"policyengine_us/m{suffix}" in problems(native) + assert "version 9.9.9, not 9.9.8" in problems(site, expected="9.9.8") + assert "no fresh bytecode-cache prefix" in problems(site, fresh_cache=False) + + +def test_stale_bytecode_cannot_run_in_place_of_the_verified_source(tmp_path): + """MOCK install, real runner: an unchecked-hash .pyc compiled from older + code sits in __pycache__ beside a source that matches RECORD. A plain + interpreter runs the stale bytecode; the runner's fresh cache prefix makes + the builder run the verified source.""" + import importlib.util + import os + import py_compile + import subprocess + import sys + + site = _fake_install(tmp_path / "stale", "9.9.9", PACKAGE) + source = site / "policyengine_us" / "__init__.py" + old = tmp_path / "old" / "__init__.py" + old.parent.mkdir() + old.write_text("X = 0\n") + py_compile.compile( + str(old), + cfile=importlib.util.cache_from_source(str(source)), + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + plain = subprocess.run( + [sys.executable, "-c", "import policyengine_us; print('X', policyengine_us.X)"], + env={**os.environ, "PYTHONPATH": str(site), "PYTHONDONTWRITEBYTECODE": "1"}, + capture_output=True, + text=True, + cwd=tmp_path, + ) + assert "X 0" in plain.stdout # the stale bytecode is what a plain import runs + found, ran = _runner(tmp_path, site, builder=_printer(tmp_path / "b")) + assert found["problems"] == [] and "X 1" in ran.stdout + + +def test_a_package_beside_the_builder_cannot_shadow_the_engine(tmp_path): + """MOCK install, real runner: a policyengine_us package in the builder's + own directory is not on the import path (-P, and runpy adds no script + directory), so the builder runs the wheel's code.""" + site = _fake_install(tmp_path / "clean", "9.9.9", PACKAGE) + beside = tmp_path / "scripts" + (beside / "policyengine_us").mkdir(parents=True) + (beside / "policyengine_us" / "__init__.py").write_text("X = -1\n") + found, ran = _runner(tmp_path, site, builder=_printer(beside)) + assert found["problems"] == [] and "X 1" in ran.stdout + + +def test_a_symlink_in_the_package_is_refused(tmp_path): + """MOCK installs, real runner: a symlinked directory (a package that would + shadow a recorded module, its files hidden from the scan) and a symlinked + file each stop the runner before the builder runs; installs must be + copies, not links (so uv's symlink link mode is not supported).""" + outside = tmp_path / "outside" / "spm" + outside.mkdir(parents=True) + (outside / "__init__.py").write_text("X = 99\n") + linked_dir = _fake_install( + tmp_path / "dir", "9.9.9", {**PACKAGE, "policyengine_us/spm.py": "S = 1\n"} + ) + (linked_dir / "policyengine_us" / "spm").symlink_to( + outside, target_is_directory=True + ) + found, ran = _runner(tmp_path, linked_dir, builder=_printer(tmp_path / "b1")) + assert ran.returncode == 3 and "policyengine_us/spm" in " ".join(found["problems"]) + linked_file = _fake_install(tmp_path / "file", "9.9.9", PACKAGE) + target = tmp_path / "outside" / "m.py" + target.write_text("Y = 2\n") + (linked_file / "policyengine_us" / "m.py").unlink() + (linked_file / "policyengine_us" / "m.py").symlink_to(target) + found, ran = _runner(tmp_path, linked_file, builder=_printer(tmp_path / "b2")) + assert ran.returncode == 3 and "symlinks in the package" in " ".join( + found["problems"] + ) + + +def test_run_takes_only_fresh_receipts_and_outputs(tmp_path, monkeypatch): + """MOCK subprocess: an existing receipt, or an out-dir already holding a + computed.csv, is refused before anything runs; a run whose builder writes + nothing new is refused after it; and a relative receipt given outside the + repository is the file the child writes and the parent reads.""" + run, calls = _fake_run_writing() + monkeypatch.setattr(timing.subprocess, "run", run) + venv = _venv(tmp_path / "venv", "9.9.9") + out = tmp_path / "pass1" + out.mkdir() + receipt = tmp_path / "receipt.json" + receipt.write_text("{}") + with pytest.raises(timing.Refusal, match="each run writes a fresh receipt"): + timing.run_on_engine(venv, "9.9.9", receipt, ["--out-dir", str(out)]) + receipt.unlink() + _computed(VALUES, out / "computed.csv") + argv = ["run", "--venv", str(venv), "--engine", "9.9.9", "--receipt", str(receipt)] + with pytest.raises(timing.Refusal, match="fresh out-dir"): + timing.main([*argv, "--", "--out-dir", str(out)]) + (out / "computed.csv").unlink() + silent, _ = _fake_run_writing(computed=False) + monkeypatch.setattr(timing.subprocess, "run", silent) + with pytest.raises(timing.Refusal, match="wrote no new"): + timing.main([*argv, "--", "--out-dir", str(out)]) + # Outside the repository, a relative receipt and out-dir are resolved once. + monkeypatch.setattr(timing.subprocess, "run", run) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + with pytest.raises(SystemExit) as stopped: + timing.main( + [ + "run", + "--venv", + str(venv), + "--engine", + "9.9.9", + "--receipt", + "r.json", + "--", + "--out-dir", + "fresh", + ] + ) + assert stopped.value.code == 1 # the MOCK builder's draft refusal + assert calls[-1][1]["PB_ENGINE_RECEIPT"] == str(elsewhere / "r.json") + assert (elsewhere / "r.json").is_file() and ( + elsewhere / "fresh/computed.csv" + ).is_file() + + +def test_builder_arguments_are_read_as_the_builder_reads_them(tmp_path, monkeypatch): + """The equals form, an abbreviation and a repeated flag (last wins) name + the same out-dir the builder would use; --computed-csv names the file the + run must write; paths are resolved here; an unknown argument is refused.""" + monkeypatch.chdir(tmp_path) + out = str(tmp_path / "b") + for argv in ( + ["--out-dir", "b"], + ["--out-dir=b"], + ["--out-d", "b"], + ["--out-dir", "a", "--out-dir=b"], + ): + canonical, computed = timing.builder_arguments([*argv, "--allow-draft"]) + assert canonical == ["--out-dir", out, "--allow-draft"], argv + assert computed == tmp_path / "b" / "computed.csv" + canonical, computed = timing.builder_arguments( + ["--actions=x.json", "--out-dir", "b", "--computed-csv", "c.csv"] + ) + assert canonical == [ + "--actions", + str(tmp_path / "x.json"), + "--out-dir", + out, + "--computed-csv", + str(tmp_path / "c.csv"), + ] + assert computed == tmp_path / "c.csv" + assert timing.builder_arguments(["--evidence-request", "r.json"])[1] is None + with pytest.raises(timing.Refusal, match="cannot take these arguments"): + timing.builder_arguments(["--out-dir", "b", "--unknown"]) + + +def test_the_replica_takes_exactly_the_builders_flags(): + """builder_arguments' parser has the builder's own flags, so argparse + resolves equals forms and abbreviations the same way in both.""" + import ast + + tree = ast.parse(timing.BUILDER.read_text()) + (main,) = [ + n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main" + ] + flags = { + call.args[0].value: any( + k.arg == "action" and getattr(k.value, "value", None) == "store_true" + for k in call.keywords + ) + for call in ast.walk(main) + if isinstance(call, ast.Call) + and getattr(call.func, "attr", None) == "add_argument" + } + assert {f for f, switch in flags.items() if switch} == set(timing.BUILDER_SWITCHES) + assert {f for f, switch in flags.items() if not switch} == set( + timing.BUILDER_VALUE_FLAGS + ) + + +def test_run_checks_the_file_the_builder_actually_writes(tmp_path, monkeypatch): + """MOCK subprocess: with the equals form an old computed.csv is still + refused, and with --computed-csv the run checks that file, not the + out-dir's.""" + + def run(command, env=None, **kw): + Path(env["PB_ENGINE_RECEIPT"]).write_text( + json.dumps({"version": "9.9.9", "files_verified": 2, "problems": []}) + ) + target = Path(command[command.index("--computed-csv") + 1]) + _computed(VALUES, target) + return type("Ran", (), {"returncode": 1, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(timing.subprocess, "run", run) + venv = _venv(tmp_path / "venv", "9.9.9") + old = tmp_path / "old" + old.mkdir() + _computed(VALUES, old / "computed.csv") + base = ["run", "--venv", str(venv), "--engine", "9.9.9"] + with pytest.raises(timing.Refusal, match="fresh out-dir"): + timing.main( + [*base, "--receipt", str(tmp_path / "r1.json"), "--", f"--out-dir={old}"] + ) + with pytest.raises(SystemExit) as stopped: + timing.main( + [ + *base, + "--receipt", + str(tmp_path / "r2.json"), + "--", + "--out-dir", + str(tmp_path / "new"), + "--computed-csv", + str(tmp_path / "c.csv"), + ] + ) + assert stopped.value.code == 1 and (tmp_path / "c.csv").is_file() From 8a7086d803f9f2bf53384d014f39455b2c198657 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:22:05 -0400 Subject: [PATCH 07/28] Accept both later builders' notes in the freeze's engine-upgrade setup check A sidecar keeps every engine_upgrade revision, so the committed snapshot (last revision 2026-10-10, built by the 2026-10-09 builder) and the snapshot this release freezes (last revision 2026-10-12) must both pass read_reference_engine_setup. UPGRADE_BUILDERS lists both notes, oldest first; UPGRADE_BUILDER is the newest. Release 20261010's freeze tests pin their mock builds to their own builder's note. Co-Authored-By: Claude Opus 5.5 --- scripts/freeze_snapshot.py | 12 +++++++++--- tests/test_freeze_haiku55.py | 5 +++-- tests/test_snapshot_artifacts.py | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index d7d26978..ff74b388 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -1505,13 +1505,19 @@ def read_reference_refresh() -> dict[str, str | int]: "reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json" ), } -UPGRADE_BUILDER = ( - "reference_audit/2026-10-09-engine-upgrade/scripts/" +# Each later builder's note, oldest first: the 2026-10-10 upgrade's, then the +# 2026-10-12 one's. Both build households as the 2026-09-29 upgrade did and pin +# the same inherited setup; a sidecar keeps every upgrade's revision, so each +# note must still read as a later builder's. +UPGRADE_BUILDERS = tuple( + f"reference_audit/{folder}/scripts/" "build_references_upgrade.py; households from " "policybench.scenarios.Scenario.to_pe_household through " "reference_audit/2026-09-28/scripts/sweep.py build_situation, as in the " "2026-09-29 upgrade" + for folder in ("2026-10-09-engine-upgrade", "2026-10-12-engine-upgrade") ) +UPGRADE_BUILDER = UPGRADE_BUILDERS[-1] def read_reference_engine_setup() -> dict[str, int]: @@ -1561,7 +1567,7 @@ def read_reference_engine_setup() -> dict[str, int]: if not isinstance(builder, str): raise SystemExit("the engine_upgrade builder note must be a string") inherited_builder = ( - builder == UPGRADE_BUILDER + builder in UPGRADE_BUILDERS and set(others) == expected | UPGRADE_SUPPORT_MODULES and any( isinstance(previous.get("builder"), str) diff --git a/tests/test_freeze_haiku55.py b/tests/test_freeze_haiku55.py index f288169f..c899945d 100644 --- a/tests/test_freeze_haiku55.py +++ b/tests/test_freeze_haiku55.py @@ -2139,7 +2139,8 @@ def _builder_setup(inherited: dict) -> dict: """The setup build_references_upgrade.py records on a later upgrade (the delta review's finding 6): the inherited conventions, composer and adapter with their paths, then latest_final.py and the sales-tax table, - each pinned to its committed bytes, and the builder's note.""" + each pinned to its committed bytes, and the builder's note (the + 2026-10-10 upgrade's: the first of freeze_snapshot.UPGRADE_BUILDERS).""" fixes = "reference_audit/2026-09-28/fixes" entries = [ {"module": e["module"], "path": f"{fixes}/{e['module']}", "sha256": e["sha256"]} @@ -2154,7 +2155,7 @@ def _builder_setup(inherited: dict) -> dict: ): digest = hashlib.sha256(base_blob(Path(path))).hexdigest() entries.append({"module": name, "path": path, "sha256": digest}) - return {"fix_modules": entries, "builder": freezer.UPGRADE_BUILDER} + return {"fix_modules": entries, "builder": freezer.UPGRADE_BUILDERS[0]} def test_the_mock_setup_is_the_real_builders(): diff --git a/tests/test_snapshot_artifacts.py b/tests/test_snapshot_artifacts.py index 5ce1228e..48304406 100644 --- a/tests/test_snapshot_artifacts.py +++ b/tests/test_snapshot_artifacts.py @@ -921,7 +921,7 @@ def test_manifest_names_the_build_the_households_came_from(): modules = {entry["module"] for entry in upgrade["fix_modules"]} conventions = {m for m in modules if m.startswith("latest_c_")} base_modules = {"latest_conventions.py", "latest_md_local_output_scope.py"} - if upgrade["builder"] == freeze_snapshot.UPGRADE_BUILDER: + if upgrade["builder"] in freeze_snapshot.UPGRADE_BUILDERS: assert modules - conventions == base_modules | ( freeze_snapshot.UPGRADE_SUPPORT_MODULES ) @@ -1330,7 +1330,7 @@ def test_the_published_upgrades_pins_are_their_committed_bytes(): sidecar = json.loads((run_dir / "reference_outputs.csv.meta.json").read_text()) upgrades = [r for r in sidecar["revisions"] if r["kind"] == "engine_upgrade"] for index, upgrade in enumerate(upgrades): - inherits = upgrade.get("builder") == freeze_snapshot.UPGRADE_BUILDER + inherits = upgrade.get("builder") in freeze_snapshot.UPGRADE_BUILDERS freeze_snapshot.verify_fix_module_pins( upgrade["fix_modules"], upgrades[:index] if inherits else [] ) From cd71b644ebe321a88ccfcbe8e7706f3de59b93a0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:22:06 -0400 Subject: [PATCH 08/28] Port the freeze tests to a release that rules nothing tests/test_freeze_musewonder.py is test_freeze_haiku55.py on release 20261010's base: 48 models, 58 exclusions (release 20261010's record, byte for byte) and 1,926 scored outputs. Defects that were defects only against a ruled record (staging release 20261010's exclusion or adjudication record) are replaced by edits that still are; the ruled-entry tests go, since the spec rules none. The new model's fingerprint is derived from the registry as validate_treatment derives it, and the committed input pins are stubbed until the board run is pinned. The installed-exclusions refusal names ruled records only when the spec rules any. Co-Authored-By: Claude Opus 5.5 --- scripts/freeze_musewonder.py | 8 +- tests/test_freeze_musewonder.py | 4326 +++++++++++++++++++++++++++++++ 2 files changed, 4332 insertions(+), 2 deletions(-) create mode 100644 tests/test_freeze_musewonder.py diff --git a/scripts/freeze_musewonder.py b/scripts/freeze_musewonder.py index 14dc2159..c24dc280 100644 --- a/scripts/freeze_musewonder.py +++ b/scripts/freeze_musewonder.py @@ -390,9 +390,13 @@ def verify_installed_exclusions( raise SystemExit( f"Staged {EXCLUSIONS} is not the installed engine upgrade's record" ) + ruled = ( + f" plus the {driver.NEW_EXCLUSIONS} ruled records ({driver.SPEC_PATH})" + if driver.NEW_EXCLUSIONS + else "" + ) raise SystemExit( - f"Staged {EXCLUSIONS} is not release 20261010's record plus the " - f"{driver.NEW_EXCLUSIONS} ruled records ({driver.SPEC_PATH}); run " + f"Staged {EXCLUSIONS} is not release 20261010's record{ruled}; run " "--step install-exclusions, then triage and export again" ) if receipt["files"].get(bound) != expected: diff --git a/tests/test_freeze_musewonder.py b/tests/test_freeze_musewonder.py new file mode 100644 index 00000000..e79eced8 --- /dev/null +++ b/tests/test_freeze_musewonder.py @@ -0,0 +1,4326 @@ +"""The Muse Wonder freeze builds nothing from an unbound or revised stage. + +Adapted from test_freeze_gpt61sol.py, with synthetic stages in tmp_path. Release +20261010's committed references, exclusions, adjudications, manifest and +serving configuration are read from git at the driver's BASE_COMMIT, never +written. No test runs the real freezer: freeze_snapshot.main is replaced in +every test, its destinations point into the test's workspace, and a module +fixture checks that the checkout's tracked snapshot files were not touched. +""" + +from __future__ import annotations + +import copy +import datetime +import functools +import hashlib +import json +import re +import shutil +import sys +import types +from dataclasses import replace +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import finish_musewonder as driver # noqa: E402 +import freeze_musewonder as release # noqa: E402 +import freeze_snapshot as freezer # noqa: E402 + +from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 +from tests import test_finish_musewonder as mock_builds # noqa: E402 + +REPO = Path(__file__).resolve().parents[1] +RUN = driver.RUN_NAME +SLUG = "musewonder" +NEW = "muse-wonder" +PINNED = driver.PINNED_INPUTS +BUNDLE = Path("publish") / RUN +SNAPSHOT_DIR = Path("paper/snapshot/20260501") +FROZEN_RUN = SNAPSHOT_DIR / "runs" / RUN +ANNOTATIONS_DIR = Path("annotations") / RUN +ADJUDICATIONS = "us_adjudications.json" +ADJUDICATIONS_PATH = ANNOTATIONS_DIR / ADJUDICATIONS +EXPLANATIONS = "us_case_reference_explanations.csv" +EXCLUSIONS = "reference_exclusions.json" +ROWS, NOTES = "us_audit_row_annotations.csv", "us_case_notes.csv" +AMENDMENTS_RECORD = f"us_{driver.AMENDMENTS}" +# A UTC day a synthetic ruled wave's decisions were written on: the committed +# spec rules nothing, so it names none. +WRITTEN_ON = "2026-10-12" +# A real incumbent, so the freeze's incumbent prediction gate compares rows. +INCUMBENT = "claude-fable-5" +# Settings for Hypothesis tests: the autouse fixtures only patch modules, so +# sharing them across examples is safe. +PROPERTY = settings( + deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture] +) + + +def examples(count: int) -> settings: + return settings(PROPERTY, max_examples=count) + + +# The synthetic stage's one judged case, an incumbent-only case kept from the +# seed, and the audit evidence the receipt must bind for it. +KEPT = "us__scenario_000__snap" +AUDIT_EVIDENCE = [Path("audit/cases.jsonl"), Path("audit/schema.json")] + [ + Path("audit/cases") / KEPT / name + for name in ("prompt.md", "verdict.json", "verdict.meta.json") +] +ANNOTATION_CSVS = [BUNDLE / "annotations" / name for name in (ROWS, NOTES)] + +# --- Git baselines --------------------------------------------------------------- + +# Captured at import: a test may replace driver.base_commit_blob, and the +# caches below must hold release 20261010's bytes whatever it does. +_BASE_COMMIT_BLOB = driver.base_commit_blob +_BASE_PAYLOAD = driver.base_payload_from_commit +_HEAD_BLOB = driver.head_blob + +# The runs root Muse Wonder's supervised run is written under (its run is +# //run); docs/musewonder/input_pins.json names it once the run +# is pinned. +RUNS = {SLUG: f"musewonder-runs/{SLUG}"} + + +@functools.cache +def base_blob(path: Path) -> bytes: + """A repository file as committed at the driver's BASE_COMMIT.""" + return _BASE_COMMIT_BLOB(Path(path)) + + +@functools.cache +def head_blob(path: Path) -> bytes: + return _HEAD_BLOB(Path(path).as_posix()) + + +@functools.cache +def _spec_text() -> str: + return (REPO / driver.SPEC_PATH).read_text() + + +def release_spec(**changes) -> dict: + """docs/musewonder/spec.json, with any changes a test makes.""" + spec = json.loads(_spec_text()) + spec.update(changes) + return spec + + +@functools.cache +def release_exclusions_text() -> str: + """The release's exclusion record: release 20261010's (nothing ruled).""" + base = json.loads(base_blob(FROZEN_RUN / EXCLUSIONS)) + return driver.exclusions_text(driver.build_release_exclusions(base, release_spec())) + + +def release_exclusions() -> list[dict]: + return json.loads(release_exclusions_text())["exclusions"] + + +@functools.cache +def _base_stats_text() -> str: + payload = _BASE_PAYLOAD() + return json.dumps(payload["countries"]["us"]["modelStats"]) + + +def base_stats() -> list[dict]: + """Release 20261010's modelStats, as the base payload in git holds them.""" + return json.loads(_base_stats_text()) + + +def base_live() -> dict: + """The part of release 20261010's payload export reads: its modelStats.""" + return {"countries": {"us": {"modelStats": base_stats()}}} + + +# --- Guards ---------------------------------------------------------------------- + +GUARDED = ( + Path("app/src/data.artifact.json"), + Path("app/src/data.versions.json"), + Path("app/src/paperSnapshot.json"), + SNAPSHOT_DIR, + ANNOTATIONS_DIR, + Path("app/.cache"), +) + + +def _tree_state() -> dict[str, tuple[int, int]]: + state = {} + for name in GUARDED: + path = REPO / name + paths = [path] if path.is_file() else sorted(path.rglob("*")) + for item in paths: + if item.is_file(): + stat = item.stat() + state[str(item)] = (stat.st_size, stat.st_mtime_ns) + return state + + +@pytest.fixture(scope="module", autouse=True) +def checkout_untouched(): + """No test in this module writes the checkout's tracked freeze outputs.""" + before = _tree_state() + yield + assert _tree_state() == before + + +# configure_freezer assigns these on the freeze_snapshot module itself. +CONFIGURED = ( + "SNAPSHOT_DATE", + "SOURCE_RUN", + "SOURCE_US", + "SOURCE_ANNOTATIONS", + "REFERENCE_META_SOURCE", + "REFERENCE_PINS", + "PUBLISHED_DASHBOARD_SOURCE", + "PUBLISHED_DASHBOARD_ARTIFACT", + "RUN_STATE_EVIDENCE", + "AUDIT_CASES_DIR", + "audit_judge_provenance", + "developer_adjudications_block", + "freeze_serving_configuration", +) + + +@pytest.fixture(autouse=True) +def no_real_freezer(monkeypatch): + """The real freezer never runs here, and what main configures is undone.""" + import pandas as pd + + def refuse(): + raise AssertionError("a test reached the real freeze_snapshot.main") + + monkeypatch.setattr(freezer, "main", refuse) + for name in CONFIGURED: + monkeypatch.setattr(freezer, name, getattr(freezer, name)) + option = "future.infer_string" + try: + saved = pd.get_option(option) + except (KeyError, pd.errors.OptionError): + saved = None + yield + if saved is not None: + pd.set_option(option, saved) + + +@pytest.fixture(autouse=True) +def spec(monkeypatch): + """The spec every driver read sees: the committed one. A test may edit + the dict.""" + value = release_spec() + monkeypatch.setattr(driver, "load_spec", lambda: copy.deepcopy(value)) + return value + + +def point_freeze_at(workspace: Path, monkeypatch) -> None: + """The freeze and the freezer both write into ``workspace`` only.""" + monkeypatch.setattr(release, "ROOT", workspace) + monkeypatch.setattr(freezer, "ROOT", workspace) + monkeypatch.setattr(freezer, "SNAPSHOT_DIR", workspace / SNAPSHOT_DIR) + monkeypatch.setattr(freezer, "RUN_DEST", workspace / FROZEN_RUN) + monkeypatch.setattr(freezer, "ANNOTATIONS_DEST", workspace / ANNOTATIONS_DIR) + + +# --- Synthetic stage ------------------------------------------------------------- + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def pinned_references(directory: Path, exclusions: str | None = None) -> Path: + """Release 20261010's four pinned reference files, from git, and the + release's exclusion record (or ``exclusions``), into ``directory``.""" + directory.mkdir(parents=True, exist_ok=True) + for name in driver.REFERENCE_FILES: + if name == EXCLUSIONS: + text = release_exclusions_text() if exclusions is None else exclusions + (directory / name).write_text(text) + else: + (directory / name).write_bytes(base_blob(FROZEN_RUN / name)) + return directory + + +def base_references(directory: Path) -> Path: + """The five reference files release 20261010 committed, from git.""" + directory.mkdir(parents=True, exist_ok=True) + for name in driver.REFERENCE_FILES: + (directory / name).write_bytes(base_blob(FROZEN_RUN / name)) + return directory + + +# The fingerprint Muse Wonder's supervised run records when it starts, as the +# registry gives it (validate_treatment's expected treatment, which the freeze +# checks the run state against), so it follows the model card. +@functools.cache +def _registry_fingerprint() -> str: + from policybench.config import MODELS, PROGRAMS + from policybench.eval_no_tools import ( + _first_request_variables, + _initial_completion_budget_tokens, + _max_repair_rounds, + _request_timeout_seconds, + _thinking_configuration, + ) + from policybench.model_cards import ( + PROMPT_CONTRACT_VERSION, + answer_contract_for, + completion_budget_ceiling_for, + explanation_chunk_size_for, + ) + from policybench.scenarios import load_scenarios_from_manifest + from policybench.spec import expand_programs_for_scenario + + provider_id = MODELS[NEW] + contract = answer_contract_for(provider_id) + scenarios = REPO / FROZEN_RUN / "scenarios.csv" + first = load_scenarios_from_manifest(scenarios)[0] + variables = _first_request_variables( + provider_id, expand_programs_for_scenario(PROGRAMS, first), True + ) + return json.dumps( + { + "fingerprint_version": 3, + "model_id": provider_id, + "answer_contract": contract, + "tool_choice_mode": "forced" if contract == "tool" else None, + "chunk_size": explanation_chunk_size_for(provider_id), + "prompt_contract_version": PROMPT_CONTRACT_VERSION, + "completion_budget_ceiling": completion_budget_ceiling_for(provider_id), + "initial_completion_budget_tokens": _initial_completion_budget_tokens( + provider_id, variables + ), + "thinking": _thinking_configuration(provider_id), + "request_timeout_seconds": _request_timeout_seconds(provider_id, env={}), + "max_repair_rounds": _max_repair_rounds(env={}), + } + ) + + +NEW_FINGERPRINT = json.loads(_registry_fingerprint()) +COLUMNS = ( + "model,scenario_id,variable,prediction,explanation,raw_response," + "provider_resolved_model,prompt_tokens,completion_tokens,error," + "estimated_cost_usd,total_cost_usd,total_tokens,elapsed_seconds," + "request_started_at,request_completed_at\n" +) +# MOCK: Muse Wonder's run file as the synthetic stage pins it: the usage +# columns its published cost, tokens and latency come from, and the request +# times the response window is dated by (2026-10-13 UTC). +NEW_RUN = COLUMNS + ( + "muse-wonder,scenario_000,snap,1200.0,Used the 2026 allotment.,{}," + "muse-wonder,4000,96,,0.0125,0.0125,4096.5,12.75,1791850000.5," + "1791850012.25\n" + "muse-wonder,scenario_001,snap,0.0,Not eligible.,{},muse-wonder," + "3900,88,,0.011,0.011,3900.25,11.5,1791850010.0,1791850021.5\n" +) +WINDOW = "2026-06-12 to 2026-10-13" +# An incumbent row the bundle's predictions hold beside Muse Wonder's. +INCUMBENT_ROW = ( + f"{INCUMBENT},scenario_000,snap,900.0,Used the 2025 allotment.,," + f"{INCUMBENT},,,,,,,,,\n" +) + + +def restamp(stage: Path) -> None: + """Write stage.json and model-provenance.json for the stage as it stands, + as prepare and install-exclusions would have.""" + names = [ + BUNDLE / "us" / name for name in (*driver.REFERENCE_FILES, "predictions.csv") + ] + names += [Path("inputs") / SLUG / name for name in PINNED] + files = {str(name): sha(stage / name) for name in names} + installed = { + "base_sha256": driver.BASE_REFERENCE_SHA256[EXCLUSIONS], + "sha256": files[str(BUNDLE / "us" / EXCLUSIONS)], + "spec_sha256": sha(REPO / driver.SPEC_PATH), + "records": driver.RELEASE_EXCLUSIONS, + } + record = { + "partial": False, + "early": False, + "base_tag": driver.BASE_TAG, + "base_commit": driver.BASE_COMMIT, + "files": files, + "exclusions_installed": installed, + } + (stage / "stage.json").write_text(json.dumps(record)) + inputs = stage / "inputs" / SLUG + state = json.loads((inputs / "run_state.json").read_text()) + provenance = { + NEW: { + "predictions_sha256": sha(inputs / "predictions.csv"), + "treatment_fingerprint": state["treatment_fingerprint"], + } + } + (stage / "model-provenance.json").write_text(json.dumps(provenance)) + + +def write_inputs(stage: Path) -> dict: + """Muse Wonder's run files, the bundle's predictions holding its rows + as prepare folds them, and the prepare-time hashes; returns the pins.""" + inputs = stage / "inputs" / SLUG + inputs.mkdir(parents=True, exist_ok=True) + (inputs / "predictions.csv").write_text(NEW_RUN) + state = { + "model": NEW, + "total": 100, + "completed": 100, + "stopped_reason": None, + "treatment_fingerprint": NEW_FINGERPRINT, + } + (inputs / "run_state.json").write_text(json.dumps(state)) + header, *rows = NEW_RUN.splitlines(keepends=True) + (stage / BUNDLE / "us/predictions.csv").write_text( + header + INCUMBENT_ROW + "".join(rows) + ) + restamp(stage) + return {SLUG: {name: sha(inputs / name) for name in PINNED}} + + +def write_audit(audit: Path) -> None: + """One judged, kept case with a bound seed verdict; one parse-only case, + which is never judged and so needs no verdict.""" + case = audit / "cases" / KEPT + case.mkdir(parents=True) + (audit / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA)) + manifest = [ + { + "case_id": KEPT, + "scenario_id": "scenario_000", + "variable": "snap", + "wrong_models": [INCUMBENT], + "parse_failure_only": False, + }, + { + "case_id": "us__scenario_001__snap", + "scenario_id": "scenario_001", + "variable": "snap", + "wrong_models": [INCUMBENT], + "parse_failure_only": True, + }, + ] + (audit / "cases.jsonl").write_text( + "".join(json.dumps(item) + "\n" for item in manifest) + ) + (case / "prompt.md").write_text("Classify these wrong answers.\n") + verdict = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": "The model applied an outdated threshold.", + "models": [ + { + "model": INCUMBENT, + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": "The model used the prior year's threshold.", + } + ], + } + (case / "verdict.json").write_text(json.dumps(verdict)) + (case / "verdict.meta.json").write_text( + json.dumps( + { + "verdict_sha256": sha(case / "verdict.json"), + "judge_model_requested": "default", + } + ) + ) + + +def case_dir_name(scenario_id: str, variable: str) -> str: + return f"us__{scenario_id}__{variable}" + + +def ruled_new_outputs() -> list[tuple[str, str]]: + """The ruled outputs with no earlier decision: none, as the spec rules + nothing.""" + spec = release_spec() + restated = {driver.spec_key(item) for item in spec["restated_adjudications"]} + return [ + driver.spec_key(item) + for item in spec["adjudications"] + if driver.spec_key(item) not in restated + ] + + +def write_ruled_verdicts(cases: Path) -> None: + """A bound Opus 5.5 verdict for each newly decided ruled output, as the + adjudicate-exclusions step reads it.""" + for index, output in enumerate(ruled_new_outputs()): + case = cases / case_dir_name(*output) + case.mkdir(parents=True, exist_ok=True) + verdict = { + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "reference_suspect": index % 2 == 0, + } + (case / "verdict.json").write_text(json.dumps(verdict)) + (case / "verdict.meta.json").write_text( + json.dumps( + { + "verdict_sha256": sha(case / "verdict.json"), + "judge_model_requested": driver.JUDGE_MODEL, + "judge_model_reported": [driver.JUDGE_MODEL], + "judged_at_utc": "2026-10-08T12:00:00+00:00", + } + ) + ) + + +def ruled_record(cases: Path) -> dict: + """The record --step adjudicate-exclusions writes on release 20261010's: + the spec rules nothing, so it is that release's record.""" + return driver.exclusion_adjudications(driver.base_adjudication_record(), cases) + + +def write_judge_verdicts(cases: Path, record: dict) -> None: + """For every other decided case, a verdict whose classes and flag are the + ones its entry records the judge giving.""" + for entry in record["adjudications"]: + case = cases / case_dir_name(entry["scenario_id"], entry["variable"]) + if (case / "verdict.json").is_file(): + continue + case.mkdir(parents=True, exist_ok=True) + verdict = { + "case_failure_source": entry["judge_failure_source"], + "case_failure_subtype": entry["judge_failure_subtype"], + "reference_suspect": bool(entry.get("judge_reference_suspect")), + } + (case / "verdict.json").write_text(json.dumps(verdict)) + + +def test_the_freeze_adds_muse_wonder_alone_under_the_driver_tag(): + assert release.NEW_MODELS == {SLUG: NEW} == driver.MODELS + assert release.BOARD_MODELS == 48 + assert (driver.RELEASE_EXCLUSIONS, driver.RELEASE_SCORED) == (58, 1926) + assert release.RUN == RUN + + +@pytest.fixture +def freeze_preflight(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + point_freeze_at(workspace, monkeypatch) + stage = workspace / "results/local/stage" + stage.mkdir(parents=True) + payload = stage / "data-board48.json" + payload.write_text(json.dumps({"countries": {"us": {"modelStats": []}}})) + evidence = [ + BUNDLE / "us" / name for name in (*driver.REFERENCE_FILES, "predictions.csv") + ] + evidence += [BUNDLE / "annotations" / name for name in driver.ANNOTATION_FILES] + evidence += [Path(driver.PROMPT_CHANGES)] + for name in evidence: + (stage / name).parent.mkdir(parents=True, exist_ok=True) + (stage / name).write_text(f"Evidence: {name.name}\n") + # install-exclusions wrote the release's record into the bundle. + (stage / BUNDLE / "us" / EXCLUSIONS).write_text(release_exclusions_text()) + # Muse Wonder's run files are the committed pins' (stubbed here), and + # the bundle's predictions hold its rows as prepare folded them. + pins = write_inputs(stage) + monkeypatch.setattr(driver, "committed_input_pins", lambda: pins) + # The input pins as HEAD commits them, which name the supervised run the + # frozen serving evidence must name (release.evidence_runs). + record = { + "inputs": {SLUG: {"run": f"{RUNS[SLUG]}/run", "sha256": pins[SLUG]}} + } + monkeypatch.setattr( + driver, + "head_blob", + lambda path: ( + json.dumps(record).encode() + if str(path) == driver.INPUT_PINS_PATH + else _HEAD_BLOB(path) + ), + ) + evidence += [Path("inputs") / SLUG / name for name in PINNED] + evidence += [Path("model-provenance.json"), Path("stage.json")] + write_audit(stage / "audit") + evidence += AUDIT_EVIDENCE + (stage / driver.PROMPT_CHANGES).write_text( + json.dumps({"added": [], "changed": [], "kept": []}) + ) + receipt = { + "release_tag": driver.RELEASE_TAG, + "base_tag": driver.BASE_TAG, + "base_sha256": driver.BASE_SHA256, + "payload_sha256": sha(payload), + "models": 48, + "partial": False, + "files": {str(name): sha(stage / name) for name in evidence}, + } + (stage / "release-ready.json").write_text(json.dumps(receipt)) + app = workspace / "app/src" + app.mkdir(parents=True) + # The live pointer and version list as release 20261010 left them. + (app / "data.artifact.json").write_bytes(base_blob(Path(release.POINTER))) + (app / "data.versions.json").write_bytes(base_blob(Path(release.VERSIONS))) + return stage, payload, receipt + + +def workspace_files(): + return { + path.relative_to(release.ROOT): path.read_bytes() + for path in release.ROOT.rglob("*") + if path.is_file() + } + + +def rewrite_receipt(stage: Path, receipt: dict) -> None: + receipt["files"] = {name: sha(stage / name) for name in receipt["files"]} + (stage / "release-ready.json").write_text(json.dumps(receipt)) + + +def test_freeze_rejects_pretty_json_before_any_workspace_mutation(freeze_preflight): + stage, payload, receipt = freeze_preflight + payload.write_text(json.dumps(json.loads(payload.read_text()), indent=2) + "\n") + receipt["payload_sha256"] = sha(payload) + (stage / "release-ready.json").write_text(json.dumps(receipt)) + before = workspace_files() + with pytest.raises(SystemExit, match="does not recombine to the freeze format"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +def test_the_default_tag_is_the_driver_release_tag(freeze_preflight, monkeypatch): + stage, _, receipt = freeze_preflight + # MOCK: another dated release, the day after the driver's, so the tag the + # freeze defaults to can only have come from the driver. + day = datetime.datetime.strptime(driver.RELEASE_TAG.rsplit("-", 1)[1], "%Y%m%d") + other = f"dashboard-data-{day + datetime.timedelta(days=1):%Y%m%d}" + assert other != driver.RELEASE_TAG + monkeypatch.setattr(driver, "RELEASE_TAG", other) + with pytest.raises(SystemExit, match=f"release tag '{other}'"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + receipt["release_tag"] = other + (stage / "release-ready.json").write_text(json.dumps(receipt)) + # Past the receipt, the spec still names the release it was written for. + with pytest.raises(SystemExit, match="names another release or base"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +def test_a_tag_that_is_not_a_dated_release_is_refused(freeze_preflight): + stage, _, _ = freeze_preflight + with pytest.raises(SystemExit): + release.main(["--stage-dir", str(stage), "--tag", "latest", "--dry-run"]) + + +def test_the_freeze_refuses_an_unbound_MOCK_engine_upgrade(freeze_preflight): + """MOCK: an engine name alone does not bind an installed build.""" + stage, _, receipt = freeze_preflight + assert not release.engine_upgrade_installed(stage) + assert not release.engine_upgrade_installed(stage / "absent") + (stage / "stage.json").write_text( + json.dumps({"references_installed": {"engine_version": "MOCK 2.40.9"}}) + ) + rewrite_receipt(stage, receipt) + assert release.engine_upgrade_installed(stage) + before = workspace_files() + for dry_run in ([], ["--dry-run"]): + with pytest.raises(SystemExit, match="does not bind the installed build"): + release.main(["--stage-dir", str(stage), *dry_run]) + assert workspace_files() == before + + +@pytest.mark.parametrize( + "defect", + [ + "changed", + "outside_stage", + "missing_hashes", + "board46", + "partial", + "tag", + "base_tag", + "base_sha256", + ], +) +def test_freeze_refuses_changed_or_unbound_evidence(freeze_preflight, defect): + stage, _, receipt = freeze_preflight + message = "Strict export receipt does not match" + if defect == "changed": + (stage / "audit/cases" / KEPT / "verdict.json").write_text("Altered\n") + message = "Staged evidence changed" + elif defect == "outside_stage": + path = release.ROOT / "app/src/data.artifact.json" + receipt["files"]["../../../app/src/data.artifact.json"] = sha(path) + message = "Staged evidence changed" + elif defect == "missing_hashes": + receipt["files"] = {} + message = "missing staged evidence hashes" + elif defect == "board46": + receipt["models"] = 47 + elif defect == "partial": + receipt["partial"] = True + elif defect == "base_tag": + receipt["base_tag"] = "dashboard-data-20260930" + message = "does not name release 20261010 as its base" + elif defect == "base_sha256": + receipt["base_sha256"] = "0" * 64 + message = "does not name release 20261010 as its base" + else: + receipt["release_tag"] = driver.BASE_TAG + (stage / "release-ready.json").write_text(json.dumps(receipt)) + before = workspace_files() + with pytest.raises(SystemExit, match=message): + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + + +@pytest.mark.parametrize( + "unbound", + [ + str(BUNDLE / "us" / EXCLUSIONS), + str(BUNDLE / "annotations" / ADJUDICATIONS), + str(BUNDLE / "annotations" / EXPLANATIONS), + f"inputs/{SLUG}/run_state.json", + f"inputs/{SLUG}/predictions.csv", + "model-provenance.json", + driver.PROMPT_CHANGES, + "stage.json", + ], +) +def test_the_receipt_must_bind_references_adjudications_and_run_state( + freeze_preflight, unbound +): + stage, _, receipt = freeze_preflight + del receipt["files"][unbound] + (stage / "release-ready.json").write_text(json.dumps(receipt)) + with pytest.raises(SystemExit, match="does not bind"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +@pytest.mark.parametrize( + "unbound", [str(path) for path in (*AUDIT_EVIDENCE, *ANNOTATION_CSVS)] +) +def test_the_receipt_must_bind_every_judged_case_and_both_annotation_csvs( + freeze_preflight, unbound +): + """Without its receipt entry, a kept verdict or sidecar could be edited + and re-hashed after export; the freeze refuses the missing entry.""" + stage, _, receipt = freeze_preflight + del receipt["files"][unbound] + (stage / "release-ready.json").write_text(json.dumps(receipt)) + before = workspace_files() + with pytest.raises(SystemExit, match=f"does not bind.*{re.escape(unbound)}"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + + +def test_a_judged_cases_transcript_must_be_bound(freeze_preflight): + """An isolated judge's transcript, which the provenance gate reads, could + otherwise be edited and re-hashed after export.""" + stage, _, receipt = freeze_preflight + transcript = Path("audit/cases") / KEPT / "claude.transcript.jsonl" + (stage / transcript).write_text("{}\n") + with pytest.raises( + SystemExit, match=f"does not bind.*{re.escape(str(transcript))}" + ): + release.main(["--stage-dir", str(stage), "--dry-run"]) + receipt["files"][str(transcript)] = sha(stage / transcript) + (stage / "release-ready.json").write_text(json.dumps(receipt)) + # Bound, it passes the receipt; the synthetic payload fails the schema. + with pytest.raises(SystemExit, match="Strict dashboard gate failed"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +def test_wording_amendments_present_in_the_stage_must_be_bound(freeze_preflight): + stage, _, _ = freeze_preflight + (stage / driver.AMENDMENTS).write_text(json.dumps({"amendments": []})) + with pytest.raises(SystemExit, match=f"does not bind.*{driver.AMENDMENTS}"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +# --- The spec -------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "field, value", + [ + ("release_tag", "dashboard-data-20261010"), + ("base_tag", "dashboard-data-20260930"), + ("base_sha256", "0" * 64), + ], +) +def test_the_freeze_refuses_a_spec_for_another_release_or_base(spec, field, value): + spec[field] = value + with pytest.raises(SystemExit, match=f"names another release or base.*{field}"): + release.release_spec(driver.RELEASE_TAG) + + +@pytest.mark.parametrize( + "field, value, message", + [ + ("snapshot_date", None, "names no snapshot_date"), + ("snapshot_date", "2026-10-32", "names no snapshot_date"), + ("snapshot_date", 20261009, "names no snapshot_date"), + ], +) +def test_the_freeze_refuses_a_spec_without_its_dates(spec, field, value, message): + if value is None: + del spec[field] + else: + spec[field] = value + with pytest.raises(SystemExit, match=message): + release.release_spec(driver.RELEASE_TAG) + + +def test_the_spec_names_this_release_and_its_snapshot_date(): + value = release.release_spec(driver.RELEASE_TAG) + assert value["snapshot_date"] == "2026-10-13" + # It rules nothing, so it names no day a ruled wave's decisions were written. + assert driver.spec_records(value) == [] + assert "adjudications_written_on" not in value + + +@pytest.mark.parametrize("written", [None, " "]) +def test_a_spec_that_rules_records_must_name_their_written_day( + spec, monkeypatch, written +): + """Were the spec to rule records, the record's date conventions would + state the day their decisions were written, so the freeze needs it.""" + monkeypatch.setattr(driver, "spec_records", lambda spec: [{"scenario_id": "x"}]) + if written is not None: + spec["adjudications_written_on"] = written + with pytest.raises(SystemExit, match="names no adjudications_written_on"): + release.release_spec(driver.RELEASE_TAG) + spec["adjudications_written_on"] = WRITTEN_ON + assert release.release_spec(driver.RELEASE_TAG)["adjudications_written_on"] == ( + WRITTEN_ON + ) + + +def test_a_spec_without_its_dates_stops_the_freeze_before_mutation( + freeze_preflight, spec +): + stage, _, _ = freeze_preflight + del spec["snapshot_date"] + before = workspace_files() + with pytest.raises(SystemExit, match="names no snapshot_date"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +# --- The installed exclusions ---------------------------------------------------- + + +def _record_installed(stage: Path, edit) -> None: + record = json.loads((stage / "stage.json").read_text()) + edit(record) + (stage / "stage.json").write_text(json.dumps(record)) + + +@pytest.mark.parametrize( + "defect, message", + [ + ("record_edited", "is not release 20261010's record; run"), + ("receipt_hash", "Staged evidence changed since strict export.*exclusions"), + ("not_installed", "stage.json does not record the release's 58"), + ("wrong_count", "stage.json does not record the release's 58"), + ("wrong_base", "stage.json does not record the release's 58"), + ("stale_files", "stage.json does not record the release's 58"), + ], +) +def test_the_freeze_refuses_a_stage_without_the_installed_exclusions( + freeze_preflight, defect, message +): + """The stage must score on the release's record, as install-exclusions + wrote it: its bytes, its receipt hash and its stage.json record.""" + stage, _, receipt = freeze_preflight + staged = stage / BUNDLE / "us" / EXCLUSIONS + if defect == "record_edited": + doc = json.loads(staged.read_text()) + doc["exclusions"][-2]["frozen_value"] += 1 + staged.write_text(driver.exclusions_text(doc)) + # Every hash follows the edit: stage.json's and the receipt's. + restamp(stage) + rewrite_receipt(stage, receipt) + elif defect == "receipt_hash": + receipt["files"][str(BUNDLE / "us" / EXCLUSIONS)] = "0" * 64 + (stage / "release-ready.json").write_text(json.dumps(receipt)) + else: + edits = { + "not_installed": lambda r: r.pop("exclusions_installed"), + "wrong_count": lambda r: r["exclusions_installed"].update(records=57), + "wrong_base": lambda r: r["exclusions_installed"].update( + base_sha256="0" * 64 + ), + "stale_files": lambda r: r["files"].update( + {str(BUNDLE / "us" / EXCLUSIONS): "0" * 64} + ), + } + _record_installed(stage, edits[defect]) + rewrite_receipt(stage, receipt) + before = workspace_files() + with pytest.raises(SystemExit, match=message): + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + + +def test_the_receipt_must_bind_the_installed_record(freeze_preflight): + """verify_receipt already refuses a receipt hash the staged file does not + have; the installed-exclusions gate checks the binding on its own too.""" + stage, _, receipt = freeze_preflight + text = release_exclusions_text() + expected = hashlib.sha256(text.encode()).hexdigest() + assert release.verify_installed_exclusions(stage, receipt, text) == expected + receipt["files"][str(BUNDLE / "us" / EXCLUSIONS)] = "0" * 64 + with pytest.raises(SystemExit, match="does not bind the installed"): + release.verify_installed_exclusions(stage, receipt, text) + + +def test_the_installed_record_is_the_one_the_live_stage_holds(): + """The live stage, read only, holds the record the freeze requires + (local: the stage is gitignored).""" + stage = REPO / "results/local/musewonder/stage" + staged = stage / BUNDLE / "us" / EXCLUSIONS + if not staged.is_file(): + pytest.skip("no live stage in this checkout") + installed = ( + json.loads((stage / "stage.json").read_text()).get("exclusions_installed") or {} + ) + if installed.get("spec_sha256") != sha(driver.ROOT / driver.SPEC_PATH): + pytest.skip("the live stage was installed under an earlier spec") + receipt = {"files": {str(BUNDLE / "us" / EXCLUSIONS): sha(staged)}} + assert ( + release.verify_installed_exclusions(stage, receipt, release_exclusions_text()) + == hashlib.sha256(release_exclusions_text().encode()).hexdigest() + ) + + +def _payload_country(exclusions: list[dict], n: int = 1926) -> dict: + return { + "referenceExclusions": [ + {"scenarioId": r["scenario_id"], "variable": r["variable"]} + for r in exclusions + ], + "modelStats": [{"model": "a", "n": n}, {"model": NEW, "n": n}], + } + + +def test_the_payload_must_list_the_releases_exclusions_in_order(): + records = release_exclusions() + release.verify_scored_outputs(_payload_country(records), records) + swapped = records[:] + swapped[0], swapped[1] = swapped[1], swapped[0] + with pytest.raises(SystemExit, match="their order"): + release.verify_scored_outputs(_payload_country(swapped), records) + with pytest.raises( + SystemExit, match="57 excluded outputs are not the release's 58" + ): + release.verify_scored_outputs(_payload_country(records[:-1]), records) + extra = records + [{"scenario_id": "scenario_017", "variable": "snap"}] + with pytest.raises(SystemExit, match="59 excluded outputs.*scenario_017"): + release.verify_scored_outputs(_payload_country(extra), records) + + +def test_every_model_must_be_scored_on_1926_outputs(): + records = release_exclusions() + country = _payload_country(records) + country["modelStats"][1]["n"] = 1920 + with pytest.raises(SystemExit, match=f"scored on 1926 outputs.*{NEW}': 1920"): + release.verify_scored_outputs(country, records) + + +# --- The staged board ------------------------------------------------------------ + +# One unadjudicated case of the synthetic payload, as two models answered it. +CASE = ("scenario_017", "snap") + + +def _case_row(prediction: float) -> dict: + return { + "prediction": prediction, + "groundTruth": 1200.0, + "scored": True, + "annotation": "Used the prior year's allotment.", + "failureSource": "llm_error", + "failureSubtype": "thresholds_rates", + "caseAnnotation": "Both models used the prior year's maximum allotment.", + "caseFailureSources": "llm_error", + "caseFailureSubtypes": "thresholds_rates", + } + + +def _without_carried_usage(board: dict) -> dict: + """``board`` as export_full_run returns it: Fable 5's usage not carried.""" + board = copy.deepcopy(board) + fable = next( + row + for row in board["countries"]["us"]["modelStats"] + if row["model"] == "claude-fable-5" + ) + fable.update(costUsd=0.0, costPerHousehold=0.0) + del fable["totalTokens"], fable["latencySeconds"] + return board + + +@pytest.fixture +def rebuilds(): + """Each bundle an export stub read: (which board, its directory, its files).""" + return [] + + +@pytest.fixture +def exports(): + """The boards the stubbed export_full_run returns: "release" for the + staged bundle, "scope" when export puts release 20261010's exclusion + record back. A test that edits a bundle input edits them as export would + rebuild them.""" + return {} + + +@pytest.fixture +def staged_board(freeze_preflight, monkeypatch, rebuilds, exports): + """A 48-row payload past the receipt, with release 20261010's snapshot in + the workspace. + + export_full_run is stubbed to return what it would for the bound bundle: + with the release's 58 exclusions (release 20261010's, nothing ruled) every + model is scored on 1,926 outputs and the incumbents' rows are that + release's, as they are with its record put back. The freeze's rebuild runs + export's own build (scope check included) on it, so an unedited payload + rebuilds, and each test that stops at a later gate still reaches it. + Like the real exporter, the stub writes data.json into the bundle it reads. + """ + import policybench.dashboard_schema + import policybench.full_run_export + + stage, payload, receipt = freeze_preflight + # A synthetic stage binds no seed; the re-derived case list is tested in + # test_finish_musewonder.py. + monkeypatch.setattr(driver, "rejudged_cases", lambda stage: frozenset()) + # The seed stage.json binds is the synthetic audit as written: its one + # judged case is kept, so its verdict must stay the seed's. + seed = driver.seed_digest(stage / "audit") + monkeypatch.setattr(driver, "load_seed", lambda stage: seed) + monkeypatch.setattr(driver, "base_payload_from_commit", base_live) + snapshot = release.ROOT / SNAPSHOT_DIR + # The snapshot release 20261010 committed, from BASE_COMMIT. + snapshot.mkdir(parents=True) + for name in ("manifest.json", "model_serving_config.json"): + (snapshot / name).write_bytes(base_blob(SNAPSHOT_DIR / name)) + base_references(release.ROOT / FROZEN_RUN) + (snapshot / "us_reference_outputs.csv").write_bytes( + base_blob(FROZEN_RUN / "reference_outputs.csv") + ) + # prepare copies the committed reference explanations; no step writes them. + (stage / BUNDLE / "annotations" / EXPLANATIONS).write_bytes( + base_blob(ANNOTATIONS_DIR / EXPLANATIONS) + ) + added = { + "model": NEW, + "condition": "no_tools", + "exact": 58.25, + "score": 71.5, + "n": driver.RELEASE_SCORED, + } + scenario, variable = CASE + cases = { + scenario: {variable: {INCUMBENT: _case_row(900.0), NEW: _case_row(950.0)}} + } + + def board(stats, exclusions): + country = { + "referenceExclusions": [ + {"scenarioId": r["scenario_id"], "variable": r["variable"]} + for r in exclusions + ], + "modelStats": stats, + "scenarioPredictions": copy.deepcopy(cases), + } + return {"countries": {"us": country}} + + released = base_stats() + assert {row["n"] for row in released} == {driver.RELEASE_SCORED} + staged = board([*released, added], release_exclusions()) + payload.write_text(json.dumps(staged)) + base_exclusions = json.loads(base_blob(FROZEN_RUN / EXCLUSIONS))["exclusions"] + scoped = board([*base_stats(), added], base_exclusions) + exports["release"] = _without_carried_usage(staged) + exports["scope"] = _without_carried_usage(scoped) + monkeypatch.setattr( + policybench.dashboard_schema, "validate_dashboard_payload", lambda *a, **k: [] + ) + + def export_full_run(run_dir, *, countries, skip_app_data): + assert countries == ["us"] and skip_app_data + run_dir = Path(run_dir) + files = { + path.relative_to(run_dir): path.read_bytes() + for path in run_dir.rglob("*") + if path.is_file() + } + # The release's record is release 20261010's, so the two exports read + # the same record; the scope export's copy is the one nested under the + # bundle's directory (release_20261006.export_payload). + which = "scope" if (run_dir.parent.parent / RUN).is_dir() else "release" + rebuilds.append((which, run_dir, files)) + (run_dir / "data.json").write_text("Fable 5's usage is not carried here.\n") + return copy.deepcopy(exports[which]) + + monkeypatch.setattr(policybench.full_run_export, "export_full_run", export_full_run) + + # The synthetic stage re-opens no case, so its provenance record lists none. + record = stage.parent / "judge_provenance.json" + record.write_text(json.dumps({"note": "None.", "counts": {}, "verdicts": []})) + monkeypatch.setattr(driver, "JUDGE_PROVENANCE", record) + + def rebind(restamp_stage=True, bind=()): + # Every hash in the stage follows the edit: the receipt's and, unless + # a test keeps them, stage.json's and model-provenance.json's. + if restamp_stage: + restamp(stage) + for path in bind: + receipt["files"][str(path)] = "" + receipt["payload_sha256"] = sha(payload) + receipt["judge_provenance"] = { + "path": driver.JUDGE_PROVENANCE_PATH, + "sha256": sha(record), + } + rewrite_receipt(stage, receipt) + + rebind() + return stage, rebind + + +@pytest.mark.parametrize("dry_run", [True, False]) +@pytest.mark.parametrize("defect", ["edited", "unbound", "other_path", "missing"]) +def test_the_freeze_refuses_a_provenance_record_export_did_not_bind( + staged_board, defect, dry_run +): + """Export binds the judge provenance record's hash in the receipt. A + record edited afterwards, one the receipt does not bind or binds under + another path, or a missing one, is refused before any workspace mutation.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + rebind() + receipt = json.loads((stage / "release-ready.json").read_text()) + if defect == "edited": + driver.JUDGE_PROVENANCE.write_text( + json.dumps({"note": "every judge ran isolated", "verdicts": []}) + ) + elif defect == "missing": + driver.JUDGE_PROVENANCE.unlink() + elif defect == "unbound": + del receipt["judge_provenance"] + else: + receipt["judge_provenance"]["path"] = "docs/elsewhere.json" + (stage / "release-ready.json").write_text(json.dumps(receipt)) + before = workspace_files() + with pytest.raises(SystemExit, match="not the judge provenance record"): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_the_freeze_refuses_incumbent_stats_edited_after_export(staged_board, dry_run): + """An incumbent's cost edited in the staged payload after export, with the + receipt's payload hash updated to match, is refused before any workspace + mutation: export's rebuild from the bound bundle does not give it.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + index, row = next( + (index, row) + for index, row in enumerate(board["countries"]["us"]["modelStats"]) + if row["model"] not in (NEW, "claude-fable-5") + ) + row["costUsd"] *= 2 + payload.write_text(json.dumps(board)) + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match="is not what export builds") as refused: + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + at = f"payload.countries.us.modelStats[{index} {row['model']}].costUsd" + assert repr(at) in str(refused.value) + assert workspace_files() == before + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_the_scope_check_refuses_an_incumbent_change_the_ten_records_do_not_explain( + staged_board, exports, dry_run +): + """An incumbent's cost changed in the bundle itself, so that export + rebuilds the staged payload, still leaves the release: with release + 20261010's exclusion record put back, that incumbent is not release + 20261010's, byte for byte.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + index, model = next( + (index, row["model"]) + for index, row in enumerate(board["countries"]["us"]["modelStats"]) + if row["model"] not in (NEW, "claude-fable-5") + ) + for target in (board, exports["release"], exports["scope"]): + row = target["countries"]["us"]["modelStats"][index] + assert row["model"] == model + row["costUsd"] *= 2 + payload.write_text(json.dumps(board)) + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match=f"incumbent modelStats drift.*{model}"): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +def _edit_new_model_scores(board: dict) -> list[str]: + stats = board["countries"]["us"]["modelStats"] + row = next(r for r in stats if r["model"] == NEW) + row.update(exact=100.0, score=100.0) + at = f"payload.countries.us.modelStats[{stats.index(row)} {NEW}]" + return [f"{at}.exact", f"{at}.score"] + + +def _edit_case_review(board: dict) -> list[str]: + scenario, variable = CASE + case = board["countries"]["us"]["scenarioPredictions"][scenario][variable] + for row in case.values(): + row.update( + caseFailureSources="needs_review", + reference_suspect=True, + wrong_model_count=999, + ) + at = f"payload.countries.us.scenarioPredictions.{scenario}.{variable}" + return [ + f"{at}.{INCUMBENT}.caseFailureSources", + f"{at}.{INCUMBENT}.reference_suspect (only staged)", + f"{at}.{INCUMBENT}.wrong_model_count (only staged)", + ] + + +def _edit_exclusion_text(board: dict) -> list[str]: + item = board["countries"]["us"]["referenceExclusions"][-2] + item["note"] = "A note no record carries." + return ["payload.countries.us.referenceExclusions[56].note (only staged)"] + + +# Edits to the staged payload, each of which passes every other gate once the +# receipt's payload hash is updated to match. +PAYLOAD_EDITS = { + "new_model_scores": _edit_new_model_scores, + "case_review": _edit_case_review, + "exclusion_text": _edit_exclusion_text, +} + + +@pytest.mark.parametrize("dry_run", [True, False]) +@pytest.mark.parametrize("edit", PAYLOAD_EDITS) +def test_the_freeze_refuses_a_payload_edited_after_export(staged_board, edit, dry_run): + """The receipt binds the payload only by a hash beside it. Muse Wonder's + exact and score set to 100, a case marked needs_review, or an + exclusion given a note no record carries, with the receipt's payload hash + updated to match, is refused before any workspace mutation: the payload + export builds from the bound bundle is not the staged one.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + paths = PAYLOAD_EDITS[edit](board) + payload.write_text(json.dumps(board)) + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match="is not what export builds") as refused: + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + for path in paths: + assert repr(path) in str(refused.value) + assert workspace_files() == before + + +def test_the_rebuild_reads_a_scratch_copy_of_the_bound_bundle( + staged_board, rebuilds, monkeypatch +): + """export_full_run writes into the bundle it reads. The rebuild hands it a + scratch copy outside the workspace holding exactly the bundle files the + receipt binds, with their bound bytes, and removes it; the scope check + exports a second copy with release 20261010's exclusion record. An + unedited payload rebuilds, carrying Fable 5's usage as export does, and + the freeze goes on to its next gate.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + rebind() + + def next_gate(stage): + raise SystemExit("Reached the verdict gate") + + monkeypatch.setattr(release, "verify_verdicts", next_gate) + receipt = json.loads((stage / "release-ready.json").read_text()) + before = workspace_files() + with pytest.raises(SystemExit, match="Reached the verdict gate"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + [(first, run_dir, files), (second, scope_dir, scope_files)] = rebuilds + assert (first, second) == ("release", "scope") + for directory in (run_dir, scope_dir): + assert directory.name == RUN and not directory.exists() + assert not directory.is_relative_to(release.ROOT) + bound = [Path(name) for name in receipt["files"]] + assert files == { + name.relative_to(BUNDLE): (stage / name).read_bytes() + for name in bound + if name.is_relative_to(BUNDLE) + } + assert len(files) == 10 + expected = {Path(name): files[Path(name)] for name in release_inputs_for_scope()} + expected[Path("us") / EXCLUSIONS] = driver.exclusions_text( + json.loads(base_blob(FROZEN_RUN / EXCLUSIONS)) + ).encode() + assert scope_files == expected + + +def release_inputs_for_scope() -> tuple[str, ...]: + from release_20261006 import EXPORT_INPUTS + + return EXPORT_INPUTS + + +def test_the_rebuild_reads_only_the_bytes_the_receipt_binds(staged_board, rebuilds): + """A bound bundle file that changes after the receipt was checked is + refused rather than exported.""" + stage, rebind = staged_board + payload = stage / "data-board48.json" + receipt = json.loads((stage / "release-ready.json").read_text()) + base = driver.base_payload_from_commit() + release.rebuild_payload(stage, receipt, payload, base) + (stage / BUNDLE / "us/predictions.csv").write_text("Edited after the check.\n") + with pytest.raises(SystemExit, match="changed since strict export.*predictions"): + release.rebuild_payload(stage, receipt, payload, base) + assert [which for which, *_ in rebuilds] == ["release", "scope"] + + +JSON_LEAVES = st.none() | st.booleans() | st.integers() | st.text(max_size=3) +JSON_LEAVES |= st.floats(allow_nan=False, allow_infinity=False) +JSON_VALUES = st.recursive( + JSON_LEAVES, + lambda inner: ( + st.lists(inner, max_size=3) + | st.dictionaries(st.sampled_from("abcd"), inner, max_size=3) + ), + max_leaves=12, +) + + +@examples(400) +@given(a=JSON_VALUES, b=JSON_VALUES, swap=st.booleans()) +def test_payload_differences_are_empty_exactly_when_the_bytes_match(a, b, swap): + """The freeze refuses on bytes and names paths from the parsed values; the + two agree, key order, 1 against 1.0, True against 1 and -0.0 included.""" + if swap and isinstance(a, dict): + b = dict(reversed(list(a.items()))) + differ = release.payload_differences(a, b) + assert (differ == []) == (json.dumps(a) == json.dumps(b)) + assert len(differ) <= 5 + + +def test_payload_differences_name_the_first_differing_paths(): + rebuilt = {"rows": [{"model": "m", "x": 1, "y": [1, 2]}], "z": True} + staged = {"z": 1, "rows": [{"model": "m", "y": [1], "x": 1.0, "w": 0}]} + assert release.payload_differences(rebuilt, staged) == [ + "payload.rows[0 m].x", + "payload.rows[0 m].y (length 2 rebuilt, 1 staged)", + "payload.rows[0 m].w (only staged)", + "payload.z", + "payload (key order)", + ] + + +@pytest.mark.parametrize("dry_run", [True, False]) +@pytest.mark.parametrize("defect", ["edited_kept", "unbound_new"]) +def test_the_freeze_refuses_a_verdict_edited_after_export( + staged_board, monkeypatch, defect, dry_run +): + """Export validates the verdicts before it writes the receipt. A kept + verdict edited afterwards, its classes kept, its sidecar and its receipt + entry re-hashed, must still be refused before any workspace mutation; so + must a verdict the seed does not carry that is bound to no prompt.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + case = stage / "audit/cases" / KEPT + if defect == "edited_kept": + verdict = json.loads((case / "verdict.json").read_text()) + verdict["rationale"] = "A rationale the judge never wrote." + verdict["models"][0]["diagnosis"] = "A diagnosis the judge never wrote." + (case / "verdict.json").write_text(json.dumps(verdict)) + meta = json.loads((case / "verdict.meta.json").read_text()) + meta["verdict_sha256"] = sha(case / "verdict.json") + (case / "verdict.meta.json").write_text(json.dumps(meta)) + message = f"carried-over verdicts differ.*{KEPT}" + else: + monkeypatch.setattr(driver, "load_seed", lambda stage: {}) + message = f"fail validation.*{KEPT}" + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match=message): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +def _double_cost(frame, rows) -> list[str]: + columns = ["estimated_cost_usd", "total_cost_usd"] + for column in columns: + frame.loc[rows, column] = (frame.loc[rows, column].astype(float) * 2).map(str) + return columns + + +def _slower(frame, rows) -> list[str]: + frame.loc[rows, "elapsed_seconds"] = "99.0" + return ["elapsed_seconds"] + + +USAGE_EDITS = {"cost": _double_cost, "elapsed_seconds": _slower} + + +def _edit_new_rows(path: Path, edit) -> list[str]: + import pandas as pd + + frame = pd.read_csv(path, dtype=str, keep_default_na=False) + columns = edit(frame, frame["model"] == NEW) + frame.to_csv(path, index=False) + return columns + + +def _republish_new_usage(stage: Path, exports: dict) -> None: + """The payload as export would rebuild it from the edited predictions: + Muse Wonder's costUsd changes, and the export stub agrees.""" + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + for target in (board, exports["release"], exports["scope"]): + stats = target["countries"]["us"]["modelStats"] + next(row for row in stats if row["model"] == NEW)["costUsd"] = 2.0 + payload.write_text(json.dumps(board)) + + +@pytest.mark.parametrize("dry_run", [True, False]) +@pytest.mark.parametrize("edit", USAGE_EDITS) +def test_the_freeze_refuses_new_model_usage_edited_in_the_bundle( + staged_board, exports, edit, dry_run +): + """Muse Wonder's cost columns doubled, or its elapsed_seconds + changed, in the bundle's predictions, with the payload rebuilt to match + and every hash in the stage updated (the receipt's, stage.json's and + model-provenance.json's), is refused before any workspace mutation: its + rows must be its pinned run file's in every column.""" + stage, rebind = staged_board + columns = _edit_new_rows(stage / BUNDLE / "us/predictions.csv", USAGE_EDITS[edit]) + _republish_new_usage(stage, exports) + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match="not the pinned run's") as refused: + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + for column in columns: + assert f"{NEW} {column} differs from the run on 2 rows" in str(refused.value) + assert workspace_files() == before + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_the_freeze_refuses_a_new_model_input_edited_with_the_bundle( + staged_board, exports, dry_run +): + """The staged run file edited together with the bundle's rows, every + stage hash following, is not the run file the committed pins name.""" + stage, rebind = staged_board + for path in ( + stage / "inputs" / SLUG / "predictions.csv", + stage / BUNDLE / "us/predictions.csv", + ): + _edit_new_rows(path, _double_cost) + _republish_new_usage(stage, exports) + rebind() + before = workspace_files() + with pytest.raises( + SystemExit, match=f"staged inputs/{SLUG}/predictions.csv is not the run file" + ): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +def _edit_json(path: Path, edit) -> None: + value = json.loads(path.read_text()) + edit(value) + path.write_text(json.dumps(value)) + + +@pytest.mark.parametrize( + "defect", + ["stale_stage_json", "unrecorded_input", "provenance_hash", "fingerprint"], +) +def test_the_freeze_rechecks_the_prepare_time_hashes(staged_board, defect): + """stage.json and model-provenance.json, both bound by the receipt, must + still describe the staged inputs.""" + stage, rebind = staged_board + if defect == "stale_stage_json": + path = stage / BUNDLE / "us/predictions.csv" + path.write_text(path.read_text().replace(INCUMBENT_ROW, INCUMBENT_ROW * 2)) + message = f"changed since prepare: {BUNDLE}/us/predictions.csv" + elif defect == "unrecorded_input": + _edit_json( + stage / "stage.json", + lambda record: record["files"].pop(f"inputs/{SLUG}/predictions.csv"), + ) + message = f"stage.json does not record.*inputs/{SLUG}/predictions.csv" + elif defect == "provenance_hash": + _edit_json( + stage / "model-provenance.json", + lambda record: record[NEW].update(predictions_sha256="0" * 64), + ) + message = "model-provenance.json disagrees with the staged run" + else: + _edit_json( + stage / "model-provenance.json", + lambda record: record[NEW]["treatment_fingerprint"].update( + request_timeout_seconds=600 + ), + ) + message = "model-provenance.json disagrees with the staged run" + rebind(restamp_stage=False) + before = workspace_files() + with pytest.raises(SystemExit, match=message): + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + + +def _publish_annotations(stage: Path) -> None: + """The row annotations and case notes triage publishes for the synthetic + audit, with a staged record that decides none of its cases.""" + from policybench.audit import collect_audit + + annotations = stage / BUNDLE / "annotations" + (annotations / ADJUDICATIONS).write_text(json.dumps({"adjudications": []})) + collected = collect_audit(stage / BUNDLE / "us", stage / "audit") + cases = collected["case"].rename( + columns={ + "case_failure_source": "case_failure_sources", + "case_failure_subtype": "case_failure_subtypes", + } + ) + for name, frame in ((ROWS, collected["row"]), (NOTES, cases)): + frame.to_csv(annotations / name, index=False) + + +@pytest.fixture +def annotated_board(staged_board, monkeypatch): + """The synthetic stage with release 20261010's references and the release's + exclusion record, and the annotations triage publishes, past every gate up + to the annotation gate; the next gate, the incumbents' predictions, stops + the freeze. The kept case is decided by no adjudication. The record gate + reads release 20261010's 77 decisions, which the synthetic audit lacks; + it is tested on its own.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + monkeypatch.setattr(release, "verify_adjudication_record", lambda *a, **k: 0) + _publish_annotations(stage) + + def next_gate(): + raise SystemExit("Reached the prediction gate") + + monkeypatch.setattr(release, "base_prediction_rows", next_gate) + rebind() + return stage, rebind + + +def test_the_annotations_triage_publishes_pass_the_annotation_gate(annotated_board): + stage, _ = annotated_board + with pytest.raises(SystemExit, match="Reached the prediction gate"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_the_freeze_refuses_a_failure_class_no_judge_gave( + annotated_board, exports, dry_run +): + """A row's failure_source and its case note's case_failure_sources changed + on the kept case, which no adjudication decides, with the payload rebuilt + to carry the new class (export's rebuild agrees) and the receipt hashes + of both CSVs and the payload updated, are refused before any workspace + mutation.""" + import pandas as pd + + stage, rebind = annotated_board + annotations = stage / BUNDLE / "annotations" + for name, column in ((ROWS, "failure_source"), (NOTES, "case_failure_sources")): + frame = pd.read_csv(annotations / name, dtype=str, keep_default_na=False) + kept = frame.scenario_id == "scenario_000" + assert list(frame.loc[kept, column]) == ["llm_error"] + frame.loc[kept, column] = "prompt_ambiguity" + frame.to_csv(annotations / name, index=False) + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + scenario, variable = CASE + for target in (board, exports["release"], exports["scope"]): + for row in target["countries"]["us"]["scenarioPredictions"][scenario][ + variable + ].values(): + row.update( + failureSource="prompt_ambiguity", caseFailureSources="prompt_ambiguity" + ) + payload.write_text(json.dumps(board)) + rebind() + before = workspace_files() + with pytest.raises( + SystemExit, + match=r"us_case_notes\.csv is not what triage builds.*" + r"us__scenario_000__snap case_failure_sources", + ): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +def test_the_freeze_refuses_a_revised_staged_reference_before_mutation(staged_board): + stage, _ = staged_board + before = workspace_files() + with pytest.raises(SystemExit, match="Staged reference_outputs.csv changed"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_a_reworded_explanation_is_refused(staged_board, dry_run): + """The judge's prompt and the payload carry the reference explanations; + this release rewords none, so a staged edit is refused before any + workspace mutation, whatever hashes follow it.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + path = stage / BUNDLE / "annotations" / EXPLANATIONS + path.write_bytes(path.read_bytes().replace(b"the ", b"thee ", 1)) + rebind() + before = workspace_files() + with pytest.raises(SystemExit, match=f"Staged {EXPLANATIONS} changed"): + release.main(["--stage-dir", str(stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before + + +def _stage_ruled_record(stage: Path, edit=None) -> dict: + """The staged record as adjudicate-exclusions writes it, its new ruled + cases' bound verdicts in the stage's audit tree; ``edit`` changes it.""" + cases = stage / "audit/cases" + write_ruled_verdicts(cases) + record = ruled_record(cases) + if edit is not None: + edit(record) + (stage / BUNDLE / "annotations" / ADJUDICATIONS).write_text( + driver.record_text(record) + ) + return record + + +def _drop_an_unruled_decision(record: dict) -> None: + record["adjudications"].remove(_unruled(record)[0]) + + +def _ruled_case_files(stage: Path) -> list[Path]: + cases = stage / "audit/cases" + return [ + path.relative_to(stage) + for output in ruled_new_outputs() + for path in sorted((cases / case_dir_name(*output)).iterdir()) + ] + + +def test_the_freeze_refuses_a_dropped_adjudication_before_mutation(staged_board): + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + _stage_ruled_record(stage, _drop_an_unruled_decision) + rebind(bind=_ruled_case_files(stage)) + before = workspace_files() + with pytest.raises(SystemExit, match="drop recorded decisions"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +def test_the_freeze_baseline_is_release_20261010_in_git_not_the_working_tree( + staged_board, +): + """A freeze that stopped after copying the staged record over the committed + one leaves them equal; the next freeze must still compare with git.""" + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + _stage_ruled_record(stage, _drop_an_unruled_decision) + rebind(bind=_ruled_case_files(stage)) + staged = stage / BUNDLE / "annotations" / ADJUDICATIONS + # What a partial freeze leaves behind: the working-tree record is the + # staged one, byte for byte. + freezer.ANNOTATIONS_DEST.mkdir(parents=True) + shutil.copyfile(staged, freezer.ANNOTATIONS_DEST / ADJUDICATIONS) + with pytest.raises(SystemExit, match="drop recorded decisions"): + release.main(["--stage-dir", str(stage), "--dry-run"]) + + +# --- A releasable board, frozen ---------------------------------------------------- + + +def _serving_from_the_registry(destination: Path) -> None: + """What freeze_serving_configuration writes in a checkout without the + historical run directories: every incumbent registry-backed, and Muse + Wonder backed by its run state.""" + base = json.loads(base_blob(SNAPSHOT_DIR / "model_serving_config.json")) + serving = copy.deepcopy(base) + for row in serving["models"].values(): + row["evidence"] = {"kind": "registry"} + serving["models"][NEW] = { + **copy.deepcopy(base["models"][INCUMBENT]), + "provider_id": NEW_FINGERPRINT["model_id"], + "evidence": {"kind": "run_state", "run": "somewhere/else"}, + } + serving["evidence_summary"] = {"registry": 47, "run_state": 1} + serving["registry_commit"] = "the freezer's commit" + destination.write_text(json.dumps(serving, indent=2, sort_keys=True) + "\n") + + +def _fake_freezer_main(calls: list, defects: set): + """Stand-in for freeze_snapshot.main: it writes what the freezer writes, + from where the freeze configured it, as freeze_run, freeze_annotations, + freeze_committed_artifacts and build_manifest do, with ``defects`` + applied, and records the configuration it found.""" + + def main() -> None: + calls.append( + { + name: getattr(freezer, name) + for name in ( + "SNAPSHOT_DATE", + "SOURCE_RUN", + "SOURCE_US", + "SOURCE_ANNOTATIONS", + "REFERENCE_META_SOURCE", + "PUBLISHED_DASHBOARD_SOURCE", + "PUBLISHED_DASHBOARD_ARTIFACT", + "RUN_STATE_EVIDENCE", + "AUDIT_CASES_DIR", + ) + } + ) + window = freezer.model_response_window( + freezer.SOURCE_US / "predictions.csv", + freezer.MODEL_RESPONSE_START, + freezer.MODEL_RESPONSE_DATE, + ) + run = freezer.RUN_DEST + if run.exists(): + shutil.rmtree(run) + run.mkdir(parents=True) + for name in ("scenarios.csv", "scenarios.csv.meta.json"): + freezer.copy_exact(freezer.SOURCE_US / name, run / name) + for name in ("reference_outputs.csv", EXCLUSIONS): + freezer.copy_exact(freezer.SOURCE_US / name, run / name) + (run / "reference_outputs.csv.meta.json").write_bytes( + freezer.REFERENCE_META_SOURCE.read_bytes() + ) + if freezer.REFERENCE_PINS is not None: + freezer.copy_exact( + freezer.SOURCE_US / "reference_outputs.csv", + freezer.SNAPSHOT_DIR / "us_reference_outputs.csv", + ) + if "other_exclusions" in defects: + doc = json.loads((run / EXCLUSIONS).read_text()) + doc["exclusions"][-2]["frozen_value"] += 1 + (run / EXCLUSIONS).write_text(driver.exclusions_text(doc)) + freezer.gzip_deterministic( + freezer.SOURCE_US / "predictions.csv", + run / "predictions.csv.gz", + stored_name="predictions.csv", + ) + serving = freezer.SNAPSHOT_DIR / "model_serving_config.json" + freezer.freeze_serving_configuration(serving) + destination = freezer.ANNOTATIONS_DEST + kept = (destination / ADJUDICATIONS).read_bytes() + shutil.rmtree(destination) + destination.mkdir(parents=True) + if "other_record" in defects: + kept = kept.replace(b"2026-10-10 UTC.", b"2026-10-11 UTC.", 1) + (destination / ADJUDICATIONS).write_bytes(kept) + for name in (ROWS, NOTES, EXPLANATIONS): + freezer.copy_exact(freezer.SOURCE_ANNOTATIONS / name, destination / name) + manifest = json.loads(base_blob(SNAPSHOT_DIR / "manifest.json")) + manifest["description"] = manifest["description"].replace( + manifest["snapshot_date"], freezer.SNAPSHOT_DATE + ) + manifest["snapshot_date"] = freezer.SNAPSHOT_DATE + manifest["reference_output_refresh"]["snapshot_date"] = freezer.SNAPSHOT_DATE + if freezer.REFERENCE_PINS is not None: + manifest["reference_output_refresh"] = freezer.read_reference_refresh() + manifest["committed_snapshot_artifacts"]["us_reference_outputs.csv"] = ( + freezer.REFERENCE_PINS["reference_outputs.csv"] + ) + manifest["scope"]["models"] = 47 if "models_47" in defects else 48 + manifest["model_response_date"] = window + records = json.loads((run / EXCLUSIONS).read_text())["exclusions"] + manifest["reference_exclusions"]["outputs"] = len(records) + manifest["reference_exclusions"]["scored_outputs_per_model"] = 1984 - len( + records + ) + manifest["published_dashboard_artifact"] = freezer.PUBLISHED_DASHBOARD_ARTIFACT + pointer = json.loads((freezer.ROOT / "app/src/data.artifact.json").read_text()) + manifest["live_dashboard_artifact"].update( + {key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes")} + ) + files = manifest["source_run_artifacts"][RUN]["files"] + if freezer.REFERENCE_PINS is not None: + files.update(freezer.REFERENCE_PINS) + files[EXCLUSIONS] = freezer.sha256_file(run / EXCLUSIONS) + files["predictions.csv.gz"] = freezer.sha256_file(run / "predictions.csv.gz") + committed = manifest["committed_snapshot_artifacts"] + committed["model_serving_config.json"] = freezer.sha256_file(serving) + audit = manifest["audit_annotation_artifacts"]["files"] + for name in (ADJUDICATIONS, ROWS, NOTES): + audit[name] = freezer.sha256_file(destination / name) + if freezer.REFERENCE_PINS is not None: + audit[EXPLANATIONS] = freezer.sha256_file(destination / EXPLANATIONS) + if "outside" in defects: + manifest["population_weight_artifact"]["sha256"] = "0" * 64 + if "pinned_reference" in defects: + files["reference_outputs.csv"] = "0" * 64 + if "explanations_pin" in defects: + audit[EXPLANATIONS] = "0" * 64 + if "window" in defects: + manifest["model_response_date"] = "2026-06-12 to 2026-10-09" + (freezer.SNAPSHOT_DIR / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ) + + return main + + +@pytest.fixture +def frozen_board(staged_board, monkeypatch): + """A releasable synthetic stage: release 20261010's references and the + release's exclusion record, the ruled record as adjudicate-exclusions + writes it, and each decided case's verdict. Gates whose inputs the + synthetic audit cannot hold (the annotations rebuilt from release + 20261010's 77 decided cases, and the applied-adjudications check) are + stubbed; they are tested above and in test_finish_musewonder.py. The + freezer's own main is a stand-in (_fake_freezer_main).""" + import pandas as pd + + import policybench.adjudications + + stage, rebind = staged_board + pinned_references(stage / BUNDLE / "us") + record = _stage_ruled_record(stage) + write_judge_verdicts(stage / "audit/cases", record) + rebind(bind=_ruled_case_files(stage)) + checked = [] + monkeypatch.setattr( + release, + "verify_annotation_amendments", + lambda annotations, amendments, audit: checked.append(amendments), + ) + monkeypatch.setattr( + policybench.adjudications, "verify_adjudications_applied", lambda *a: None + ) + staged_rows = pd.read_csv(stage / BUNDLE / "us/predictions.csv") + incumbent_rows = staged_rows[staged_rows["model"] != NEW].reset_index(drop=True) + monkeypatch.setattr(release, "base_prediction_rows", lambda: incumbent_rows) + monkeypatch.setattr( + freezer, "freeze_serving_configuration", _serving_from_the_registry + ) + monkeypatch.setattr(freezer, "_serving_registry_commit", lambda payload: "f" * 40) + calls, defects = [], set() + monkeypatch.setattr(freezer, "main", _fake_freezer_main(calls, defects)) + return types.SimpleNamespace( + stage=stage, + rebind=rebind, + record=record, + calls=calls, + defects=defects, + checked=checked, + ) + + +def test_a_dry_run_of_a_releasable_stage_writes_nothing(frozen_board, capsys): + stage = frozen_board.stage + before = workspace_files() + release.main(["--stage-dir", str(stage), "--dry-run"]) + assert workspace_files() == before + assert frozen_board.calls == [] + out = capsys.readouterr().out + assert f"Validated local release inputs: {driver.RELEASE_TAG}, 48 models" in out + assert "58 exclusions (0 new)" in out + assert f"model responses {WINDOW}; snapshot 2026-10-13; nothing written" in out + + +def _live_version(versions: dict) -> dict: + return next(v for v in versions["versions"] if v["id"] == versions["default"]) + + +def test_the_freeze_writes_the_release(frozen_board, capsys): + """The pointer, the version list, the frozen run's references and + 58-record exclusion record (release 20261010's), the annotations with the + staged record, whose date conventions name no new wave, the wording + amendments record, the serving rows, and the release's predictions + asset.""" + stage = frozen_board.stage + payload = stage / "data-board48.json" + release.main(["--stage-dir", str(stage)]) + assert f"Built {driver.RELEASE_TAG} locally" in capsys.readouterr().out + root = release.ROOT + tag = driver.RELEASE_TAG + pointer = { + "version": 1, + "repo": "PolicyEngine/policybench", + "tag": tag, + "asset": "dashboard-data.json", + "url": ( + "https://github.com/PolicyEngine/policybench/releases/download/" + f"{tag}/dashboard-data.json" + ), + "sha256": sha(payload), + "bytes": payload.stat().st_size, + } + assert (root / release.POINTER).read_text() == ( + json.dumps(pointer, indent=2, sort_keys=True) + "\n" + ) + versions = json.loads((root / release.VERSIONS).read_text()) + previous = json.loads(base_blob(Path(release.VERSIONS))) + live = _live_version(versions) + assert live["snapshotLabel"] == "Snapshot 2026-10-13" + assert live["description"].endswith(" - 48 models") + assert ( + "the 58 excluded outputs keep the values they were decided on: 38 from " + "policyengine-us 1.755.4, 18 from 2.15.17, 2 from 2.38.6" in live["description"] + ) + others = [v for v in versions["versions"] if v is not live] + assert others == [v for v in previous["versions"] if v["id"] != previous["default"]] + # The frozen run: release 20261010's references and the release's record. + frozen = root / FROZEN_RUN + assert (frozen / EXCLUSIONS).read_text() == release_exclusions_text() + assert len(json.loads((frozen / EXCLUSIONS).read_text())["exclusions"]) == 58 + for name in release.PINNED_REFERENCES: + assert (frozen / name).read_bytes() == base_blob(FROZEN_RUN / name) + # The annotations: the staged record and CSVs, and the amendments record. + annotations = root / ANNOTATIONS_DIR + assert sorted(p.name for p in annotations.iterdir()) == sorted( + (ADJUDICATIONS, ROWS, NOTES, EXPLANATIONS, AMENDMENTS_RECORD) + ) + staged_annotations = stage / BUNDLE / "annotations" + for name in (ADJUDICATIONS, ROWS, NOTES, EXPLANATIONS): + assert (annotations / name).read_bytes() == ( + staged_annotations / name + ).read_bytes() + committed = json.loads((annotations / ADJUDICATIONS).read_text()) + assert committed == frozen_board.record + conventions = committed["date_conventions"] + base = json.loads(base_blob(ADJUDICATIONS_PATH)) + assert conventions == base["date_conventions"] + assert ( + "(2026-09-05, 2026-09-22, 2026-09-29, 2026-10-05, 2026-10-06 or 2026-10-10)" + in conventions + ) + assert conventions.count("and were written on 2026-10-10 UTC.") == 1 + from policybench.adjudications import excluded_case_keys, parse_adjudications + + entries = parse_adjudications(committed, "frozen") + assert excluded_case_keys(entries) == { + (r["scenario_id"], r["variable"]) for r in release_exclusions() + } + assert (annotations / AMENDMENTS_RECORD).read_bytes() == base_blob( + ANNOTATIONS_DIR / AMENDMENTS_RECORD + ) + # The serving rows: release 20261010's, and Muse Wonder's run state. + serving = json.loads( + (root / SNAPSHOT_DIR / "model_serving_config.json").read_text() + ) + previous_serving = json.loads(base_blob(SNAPSHOT_DIR / "model_serving_config.json")) + for model, row in previous_serving["models"].items(): + assert serving["models"][model] == row + assert serving["models"][NEW]["evidence"] == { + "kind": "run_state", + "run": f"{RUNS[SLUG]}", + } + assert serving["evidence_summary"] == { + "registry": previous_serving["evidence_summary"]["registry"], + "run_state": previous_serving["evidence_summary"]["run_state"] + 1, + } + assert serving["registry_commit"] == "f" * 40 + # The freezer read the stage, as configured. + [configured] = frozen_board.calls + assert configured["SNAPSHOT_DATE"] == "2026-10-13" + assert configured["SOURCE_RUN"] == stage / BUNDLE + assert configured["SOURCE_US"] == stage / BUNDLE / "us" + assert configured["SOURCE_ANNOTATIONS"] == staged_annotations + assert configured["REFERENCE_META_SOURCE"] == ( + stage / BUNDLE / "us/reference_outputs.csv.meta.json" + ) + assert configured["PUBLISHED_DASHBOARD_SOURCE"] == payload + assert configured["PUBLISHED_DASHBOARD_ARTIFACT"] == { + key: pointer[key] for key in ("tag", "asset", "url", "sha256", "bytes") + } + assert configured["RUN_STATE_EVIDENCE"] == { + NEW: str(stage / "inputs" / SLUG / "run_state.json") + } + assert configured["AUDIT_CASES_DIR"] == stage / "audit/cases" + assert frozen_board.checked == [[]] + # The release assets beside the stage, and the dashboard's local cache. + assert (stage / "predictions.csv.gz").read_bytes() == ( + frozen / "predictions.csv.gz" + ).read_bytes() + cache = root / "app/.cache" / f"dashboard-data-{sha(payload)[:16]}.json" + assert cache.read_bytes() == payload.read_bytes() + + +def test_freezing_twice_writes_the_same_files(frozen_board): + """A second freeze of the same stage finds this release's committed + record and pins where release 20261010's were, passes, and writes the + same bytes.""" + stage = frozen_board.stage + release.main(["--stage-dir", str(stage)]) + first = workspace_files() + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == first + + +@pytest.mark.parametrize( + "defect, message", + [ + ("other_exclusions", "frozen reference reference_exclusions.json does not"), + ("other_record", f"frozen {ADJUDICATIONS} is not the staged one"), + ("outside", r"changes outside the release: \['population_weight_artifact"), + ("pinned_reference", r"outside the release: \['source_run_artifacts\..*"), + ("explanations_pin", f"outside the release.*files.{EXPLANATIONS}"), + ("models_47", "does not state the release.*'scope.models': 47"), + ("window", "does not state the release.*model_response_date"), + ], +) +def test_the_freeze_refuses_what_the_freezer_wrote_outside_the_release( + frozen_board, defect, message +): + frozen_board.defects.add(defect) + with pytest.raises(SystemExit, match=message): + release.main(["--stage-dir", str(frozen_board.stage)]) + + +def test_an_incumbent_prediction_change_is_refused_before_mutation( + frozen_board, monkeypatch +): + import pandas as pd + + stage = frozen_board.stage + rows = release.base_prediction_rows().copy() + rows.loc[rows["model"] == INCUMBENT, "prediction"] = 901.0 + monkeypatch.setattr(release, "base_prediction_rows", lambda: rows) + assert isinstance(rows, pd.DataFrame) + before = workspace_files() + with pytest.raises(SystemExit, match=f"Incumbent prediction evidence.*{INCUMBENT}"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +@pytest.mark.parametrize( + "edit", + [ + {"completed": 99}, + {"total": 99, "completed": 99}, + {"stopped_reason": "budget"}, + {"model": "claude-haiku-5"}, + {"treatment_fingerprint": None}, + ], +) +def test_an_incomplete_run_state_is_refused_before_mutation( + frozen_board, monkeypatch, edit +): + """The pinned run itself incomplete, stopped, another model's or without + its treatment, every pin and hash following it, is refused before any + workspace mutation.""" + stage = frozen_board.stage + inputs = stage / "inputs" / SLUG + _edit_json(inputs / "run_state.json", lambda state: state.update(edit)) + pins = {SLUG: {name: sha(inputs / name) for name in PINNED}} + monkeypatch.setattr(driver, "committed_input_pins", lambda: pins) + frozen_board.rebind() + before = workspace_files() + with pytest.raises(SystemExit, match="Missing completed run or treatment"): + release.main(["--stage-dir", str(stage)]) + assert workspace_files() == before + + +# --- What the freezer wrote ---------------------------------------------------- + + +@pytest.mark.parametrize( + "defect, message", + [ + ("extra_file", "Frozen annotation files are not the release's"), + ("named_wave", "beyond naming this release's waves"), + ("other_window", "does not state the release.*model_response_date"), + ("other_date", "does not state the release.*snapshot_date"), + ], +) +def test_verify_frozen_checks_what_the_freezer_wrote(frozen_board, defect, message): + """After a freeze, verify_frozen refuses an annotations directory holding + anything else, a frozen record (equal to the staged one) whose + conventions name a wave the release did not add, and a manifest that + states another window or snapshot date than the freeze derived.""" + stage = frozen_board.stage + release.main(["--stage-dir", str(stage)]) + annotations = release.ROOT / ANNOTATIONS_DIR + window, snapshot_date = WINDOW, "2026-10-13" + if defect == "extra_file": + (annotations / "notes.txt").write_text("Left behind.\n") + elif defect == "named_wave": + text = (annotations / ADJUDICATIONS).read_text() + named = text.replace(" or 2026-10-10)", ", 2026-10-10 or 2026-10-13)", 1) + assert named != text + for path in (annotations, stage / BUNDLE / "annotations"): + (path / ADJUDICATIONS).write_text(named) + elif defect == "other_window": + window = "2026-06-12 to 2026-10-09" + else: + snapshot_date = "2026-10-14" + pointer = json.loads((release.ROOT / release.POINTER).read_text()) + serving = json.loads(base_blob(SNAPSHOT_DIR / "model_serving_config.json")) + arguments = dict( + stage=stage, + snapshot=release.ROOT / SNAPSHOT_DIR, + annotations=annotations, + exclusions_sha256=hashlib.sha256( + release_exclusions_text().encode() + ).hexdigest(), + pointer=pointer, + window=window, + snapshot_date=snapshot_date, + previous_serving=serving, + incumbents=set(serving["models"]), + runs={SLUG: f"{RUNS[SLUG]}"}, + ) + with pytest.raises(SystemExit, match=message): + release.verify_frozen(**arguments) + + +@functools.cache +def _manifest_text() -> str: + return base_blob(SNAPSHOT_DIR / "manifest.json").decode() + + +def manifest_leaves(value, path=()): + if isinstance(value, dict) and value: + for key, item in value.items(): + yield from manifest_leaves(item, (*path, key)) + elif isinstance(value, list) and value: + for index, item in enumerate(value): + yield from manifest_leaves(item, (*path, index)) + else: + yield path + + +@functools.cache +def _leaves() -> tuple: + return tuple(manifest_leaves(json.loads(_manifest_text()))) + + +def _set(value, path, new): + for part in path[:-1]: + value = value[part] + value[path[-1]] = new + + +def _allowed(path) -> bool: + return path not in release.MANIFEST_PINNED and any( + path[: len(prefix)] == prefix for prefix in release.MANIFEST_CHANGES + ) + + +@examples(300) +@given(picks=st.sets(st.integers(min_value=0, max_value=10**6), max_size=6)) +def test_the_frozen_manifest_changes_only_where_the_release_may(picks): + """For any set of changed leaves of release 20261010's manifest, the + problems are exactly the changed leaves outside the release's paths or at + a pinned reference, each once.""" + leaves = _leaves() + before = json.loads(_manifest_text()) + after = copy.deepcopy(before) + changed = {leaves[pick % len(leaves)] for pick in picks} + for index, path in enumerate(sorted(changed, key=repr)): + _set(after, path, ["changed", index]) + problems = release.manifest_problems(before, after) + assert len(problems) == len(set(problems)) + assert set(problems) == {path for path in changed if not _allowed(path)} + + +def test_the_manifest_paths_the_release_may_change(): + before = json.loads(_manifest_text()) + files = ("source_run_artifacts", RUN, "files") + allowed = [ + ("scope", "models"), + ("snapshot_date",), + ("model_response_date",), + ("reference_exclusions", "outputs"), + (*files, EXCLUSIONS), + (*files, "data.json.gz"), + ("audit_annotation_artifacts", "files", ADJUDICATIONS), + ] + refused = [ + ("scope", "households"), + ("population_weight_artifact", "sha256"), + ("committed_snapshot_artifacts", "us_reference_outputs.csv"), + ("source_run_artifacts", RUN, "prompt_payload_sha256"), + *((*files, name) for name in release.PINNED_REFERENCES), + ("audit_annotation_artifacts", "files", EXPLANATIONS), + ] + for path in allowed + refused: + after = copy.deepcopy(before) + _set(after, path, "changed") + expected = [] if path in allowed else [path] + assert release.manifest_problems(before, after) == expected, path + after = copy.deepcopy(before) + after["a_new_block"] = {} + assert release.manifest_problems(before, after) == [("a_new_block",)] + + +# --- MOCK installed engine upgrades ----------------------------------------------- + + +def _builder_setup() -> dict: + """The setup build_references_upgrade.py records on a later upgrade (the + delta review's finding 6): the 2026-09-29 upgrade's conventions, composer + and adapter with their paths, then latest_final.py and the sales-tax + table, each pinned to its committed bytes (the list the 2026-10-10 + revision pins), and the 2026-10-12 builder's note.""" + meta = json.loads(base_blob(FROZEN_RUN / driver.META_NAME)) + first = next(r for r in meta["revisions"] if r["kind"] == "engine_upgrade") + fixes = "reference_audit/2026-09-28/fixes" + entries = [ + {"module": e["module"], "path": f"{fixes}/{e['module']}", "sha256": e["sha256"]} + for e in first["fix_modules"] + ] + for name, path in ( + ("latest_final.py", f"{fixes}/latest_final.py"), + ( + "r19_irs_sales_tax_2025.json", + "reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json", + ), + ): + digest = hashlib.sha256(base_blob(Path(path))).hexdigest() + entries.append({"module": name, "path": path, "sha256": digest}) + return {"fix_modules": entries, "builder": freezer.UPGRADE_BUILDER} + + +def test_the_later_builders_setup_is_the_one_release_20261010_pins(): + """The 2026-10-12 builder records the module list the 2026-10-10 revision + pins (build_references_upgrade.fix_module_pins refuses any other), under + its own note.""" + last = json.loads(base_blob(FROZEN_RUN / driver.META_NAME))["revisions"][-1] + assert last["fix_modules"] == _builder_setup()["fix_modules"] + assert last["builder"] == freezer.UPGRADE_BUILDERS[0] != freezer.UPGRADE_BUILDER + + +def test_the_mock_setup_is_the_real_builders(): + """The MOCK builds' setup is the one every real build recorded: the + committed snapshot's engine upgrade from 2.38.6 (once a freeze commits + one, so the check runs everywhere), and the release build on this machine + (mock_builds.REAL_BUILD, under results/local).""" + real = [] + committed = json.loads((REPO / FROZEN_RUN / driver.META_NAME).read_text()) + last = committed["revisions"][-1] + if ( + last.get("kind") == "engine_upgrade" + and last.get("previous_engine_version") == driver.BASE_ENGINE + ): + real.append(("committed snapshot", last)) + local = [mock_builds.REAL_BUILD / "build"] + for build in local: + meta = build / driver.META_NAME + if meta.is_file(): + real.append((str(build), json.loads(meta.read_text())["revisions"][-1])) + if not real: + pytest.skip("no real engine-upgrade build is committed or on this machine") + for where, revision in real: + assert _builder_setup() == { + "fix_modules": revision["fix_modules"], + "builder": revision["builder"], + }, where + + +def _install_MOCK_build( + stage: Path, receipt: dict, build, *, with_inherited_setup: bool = True +) -> driver.Upgrade: + """MOCK: reproduce install-references' file and stage.json bindings. + + The synthetic audit is intentionally smaller than the committed board, so + rendering a real install is unsuitable here. The build itself is gated + against the real release 20261010 by load_build, without replacing it. + """ + # MOCK values/narratives take the real builder's setup on the inherited + # conventions; the real freezer still validates the sidecar's setup note + # and pins before its first write. + if with_inherited_setup: + mock_builds.edit_json( + build.built / driver.META_NAME, + lambda meta: meta["revisions"][-1].update(_builder_setup()), + ) + upgrade = mock_builds.load(build) + copy_dir = stage / driver.BUILD_COPY + copy_dir.mkdir(exist_ok=True) + for name in driver.BUILD_COPY_FILES: + source = { + driver.EXPLANATIONS_NAME: build.explanations, + driver.ACTIONS_NAME: build.actions, + }.get(name, build.built / name) + shutil.copyfile(source, copy_dir / name) + receipt["files"][str(Path(driver.BUILD_COPY) / name)] = sha(copy_dir / name) + prepared = json.loads((stage / "stage.json").read_text()) + for target in (stage / "scoring", stage / BUNDLE / "us"): + target.mkdir(exist_ok=True) + for name in driver.UPGRADED_FILES: + shutil.copyfile(copy_dir / name, target / name) + for name in ("scenarios.csv", "scenarios.csv.meta.json"): + shutil.copyfile(stage / BUNDLE / "us" / name, stage / "scoring" / name) + shutil.copyfile( + copy_dir / EXPLANATIONS, stage / BUNDLE / "annotations" / EXPLANATIONS + ) + for name in driver.UPGRADED_FILES: + prepared["files"][str(BUNDLE / "us" / name)] = upgrade.sha256[name] + prepared["files"][str(BUNDLE / "annotations" / EXPLANATIONS)] = upgrade.sha256[ + EXPLANATIONS + ] + prepared["references_installed"] = driver.references_record(upgrade, None) + driver.bind_build_exclusions(stage, prepared, upgrade) + record = driver.exclusion_adjudications( + driver.base_adjudication_record(), + stage / "audit/cases", + upgrade.regenerated_ruled, + ) + record, dropped = driver.drop_regenerated_adjudications(record, upgrade) + prepared["adjudications_dropped"] = dropped + (stage / "stage.json").write_text(json.dumps(prepared)) + (stage / BUNDLE / "annotations" / ADJUDICATIONS).write_text( + driver.record_text(record) + ) + rewrite_receipt(stage, receipt) + return upgrade + + +@pytest.fixture +def MOCK_upgraded_board(frozen_board, exports, spec, tmp_path): + """MOCK: a releasable upgraded board, four release 20261010 engine defects + and WI_042 fixed. + + It newly excludes nothing, so the spec's upgrade_adjudications and its + triage decisions are left out. Each regenerated record has a committed + decision whose drop must be bound and checked. + """ + spec.pop("upgrade_adjudications", None) + spec.pop("triage_adjudications", None) + build = mock_builds.real_build( + tmp_path / "MOCK-build", + regenerated={**mock_builds.MOCK_REGENERATED, mock_builds.WI_042: 0.0}, + added={}, + ) + stage = frozen_board.stage + receipt = json.loads((stage / "release-ready.json").read_text()) + upgrade = _install_MOCK_build(stage, receipt, build) + records = json.loads(upgrade.exclusions_text)["exclusions"] + payload = stage / "data-board48.json" + board = json.loads(payload.read_text()) + for target in (board, exports["release"]): + country = target["countries"]["us"] + country["referenceExclusions"] = [ + {"scenarioId": r["scenario_id"], "variable": r["variable"]} for r in records + ] + for row in country["modelStats"]: + row["n"] = upgrade.scored_outputs + payload.write_text(json.dumps(board)) + + def rebind(): + receipt["payload_sha256"] = sha(payload) + rewrite_receipt(stage, receipt) + + rebind() + frozen_board.upgrade = upgrade + frozen_board.build = build + frozen_board.receipt = receipt + frozen_board.rebind = rebind + frozen_board.record = json.loads( + (stage / BUNDLE / "annotations" / ADJUDICATIONS).read_text() + ) + return frozen_board + + +def test_MOCK_installed_upgrade_is_gated_and_accepted(MOCK_upgraded_board): + """MOCK: retained build, installed copies and dropped decisions all agree.""" + board = MOCK_upgraded_board + assert release.verify_upgrade(board.stage, board.receipt) == board.upgrade + assert board.upgrade.records == 53 and board.upgrade.scored_outputs == 1931 + assert board.upgrade.regenerated_base == { + *mock_builds.MOCK_REGENERATED, + mock_builds.WI_042, + } + assert board.upgrade.regenerated_ruled == frozenset() + + +@pytest.mark.parametrize("name", driver.BUILD_COPY_FILES) +def test_MOCK_export_must_bind_every_installed_build_file(MOCK_upgraded_board, name): + """MOCK: stage.json's build pins cannot replace each export receipt entry.""" + board = MOCK_upgraded_board + del board.receipt["files"][str(Path(driver.BUILD_COPY) / name)] + with pytest.raises(SystemExit, match="does not bind the installed build"): + release.verify_upgrade(board.stage, board.receipt) + + +@pytest.mark.parametrize("name", driver.BUILD_COPY_FILES) +def test_MOCK_export_must_bind_the_installed_build_bytes(MOCK_upgraded_board, name): + """MOCK: an entry with another digest is not an export binding.""" + board = MOCK_upgraded_board + board.receipt["files"][str(Path(driver.BUILD_COPY) / name)] = "0" * 64 + with pytest.raises(SystemExit, match="does not bind the installed build's bytes"): + release.verify_upgrade(board.stage, board.receipt) + + +@examples(30) +@given( + name=st.sampled_from(driver.BUILD_COPY_FILES), + position=st.integers(min_value=0, max_value=10**6), + delta=st.integers(min_value=1, max_value=255), +) +def test_MOCK_any_single_byte_change_in_the_installed_build_is_refused( + MOCK_upgraded_board, name, position, delta +): + """MOCK: every nonzero one-byte mutation breaks the retained build pin.""" + board = MOCK_upgraded_board + path = board.stage / driver.BUILD_COPY / name + original = path.read_bytes() + edited = bytearray(original) + edited[position % len(edited)] ^= delta + path.write_bytes(edited) + try: + with pytest.raises(SystemExit, match="installed build's.*changed"): + release.verify_upgrade(board.stage, board.receipt) + finally: + path.write_bytes(original) + + +@pytest.mark.parametrize( + "field, value", + [ + ("engine_version", "MOCK other engine"), + ("records", 58), + ("changed", []), + ("regenerated_base", []), + ("added", [["MOCK", "snap"]]), + ("spec_sha256", "0" * 64), + ("superseded", None), + ("superseded", ["MOCK malformed"]), + ], +) +def test_MOCK_installed_build_receipt_must_be_the_drivers_record( + MOCK_upgraded_board, field, value +): + """MOCK: a re-hashed stage.json must still describe the gated build.""" + board = MOCK_upgraded_board + _record_installed( + board.stage, lambda r: r["references_installed"].update({field: value}) + ) + with pytest.raises( + SystemExit, match="installed build.*records|installed build's record" + ): + release.verify_upgrade(board.stage, board.receipt) + + +@pytest.mark.parametrize("defect", ["absent", "extra", "duplicate", "rewritten"]) +def test_MOCK_dropped_decisions_must_be_exactly_the_regenerated_base_records( + MOCK_upgraded_board, defect +): + """MOCK: each recorded drop must be release 20261010's actual decision.""" + board = MOCK_upgraded_board + + def edit(record): + dropped = record["adjudications_dropped"] + if defect == "absent": + del record["adjudications_dropped"] + elif defect == "extra": + dropped.append({"case_id": "us__MOCK_extra__snap", "entry": {}}) + elif defect == "duplicate": + dropped.append(copy.deepcopy(dropped[0])) + else: + dropped[0]["entry"]["reasoning"] += " MOCK rewrite." + + _record_installed(board.stage, edit) + with pytest.raises(SystemExit, match="dropped adjudications|dropped decisions"): + release.verify_upgrade(board.stage, board.receipt) + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_MOCK_changed_installed_build_is_refused_before_any_write( + MOCK_upgraded_board, dry_run +): + """MOCK: changing a retained trace is refused before workspace mutation.""" + board = MOCK_upgraded_board + (board.stage / driver.BUILD_COPY / mock_builds.TRACES_NAME).write_text( + "MOCK changed\n" + ) + before = workspace_files() + with pytest.raises( + SystemExit, match="Staged evidence changed|installed build's.*changed" + ): + release.main(["--stage-dir", str(board.stage)] + ["--dry-run"] * dry_run) + assert workspace_files() == before and board.calls == [] + + +@pytest.mark.parametrize("target", ["scoring", "bundle", "explanations"]) +def test_MOCK_installed_build_copies_must_still_match(MOCK_upgraded_board, target): + """MOCK: scoring, publication and explanations cannot diverge from the build.""" + board = MOCK_upgraded_board + path = { + "scoring": board.stage / "scoring/reference_outputs.csv", + "bundle": board.stage / BUNDLE / "us/reference_outputs.csv", + "explanations": board.stage / BUNDLE / "annotations" / EXPLANATIONS, + }[target] + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(SystemExit, match="not the installed build's"): + release.verify_upgrade(board.stage, board.receipt) + + +@pytest.mark.parametrize("name", ["scenarios.csv", "scenarios.csv.meta.json"]) +@pytest.mark.parametrize("defect", ["edited", "missing"]) +def test_MOCK_scoring_scenarios_stay_pinned_to_release_20261010( + MOCK_upgraded_board, name, defect +): + """MOCK: an upgrade never revises the stage's household draw or its sidecar.""" + board = MOCK_upgraded_board + path = board.stage / "scoring" / name + if defect == "missing": + path.unlink() + else: + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(SystemExit, match="scoring.*scenarios.*release 20261010"): + release.verify_upgrade(board.stage, board.receipt) + + +@pytest.mark.parametrize( + "defect", + [ + "missing_bundle_field", + "zero_conventions", + "13_conventions", + "unlisted_module", + "empty_regeneration_timestamp", + ], +) +def test_MOCK_unsupported_manifest_sidecar_is_refused_before_any_write( + MOCK_upgraded_board, tmp_path, defect +): + """MOCK: a gated build must also support the freezer's truthful manifest note.""" + board = MOCK_upgraded_board + build = mock_builds.real_build( + tmp_path / f"MOCK-{defect}", + regenerated={**mock_builds.MOCK_REGENERATED, mock_builds.WI_042: 0.0}, + added={}, + ) + def edit(meta): + revision = meta["revisions"][-1] + revision.update(copy.deepcopy(_builder_setup())) + if defect == "missing_bundle_field": + meta["policyengine_bundles"]["us"].pop("policyengine_version") + elif defect == "empty_regeneration_timestamp": + meta["regenerated_at_utc"] = "" + elif defect == "unlisted_module": + revision["fix_modules"].append( + {"module": "MOCK_unknown.py", "sha256": "0" * 64} + ) + else: + other = [ + item + for item in revision["fix_modules"] + if not item["module"].startswith(freezer.CONVENTION_MODULE_PREFIX) + ] + count = 0 if defect == "zero_conventions" else 13 + revision["fix_modules"] = other + [ + {"module": f"latest_c_MOCK_{i}.py", "sha256": "0" * 64} + for i in range(count) + ] + + mock_builds.edit_json(build.built / driver.META_NAME, edit) + _install_MOCK_build(board.stage, board.receipt, build, with_inherited_setup=False) + before = workspace_files() + with pytest.raises(SystemExit, match="metadata|convention|fix_modules|timestamp"): + release.main(["--stage-dir", str(board.stage)]) + assert workspace_files() == before and board.calls == [] + + +def test_MOCK_committed_top_level_reference_change_is_refused_before_any_write( + MOCK_upgraded_board, +): + """MOCK: the top-level snapshot reference must also be baseline or installed.""" + board = MOCK_upgraded_board + path = release.ROOT / SNAPSHOT_DIR / "us_reference_outputs.csv" + path.write_bytes(path.read_bytes() + b" ") + before = workspace_files() + with pytest.raises(SystemExit, match="Committed us_reference_outputs.csv changed"): + release.main(["--stage-dir", str(board.stage)]) + assert workspace_files() == before and board.calls == [] + + +@pytest.mark.parametrize( + "field, value", + [ + ("source", "MOCK wrong"), + ("engine_version", "MOCK wrong"), + ("records", 58), + ("spec_sha256", "0" * 64), + ], +) +def test_MOCK_exclusion_install_receipt_must_match_the_build( + MOCK_upgraded_board, field, value +): + """MOCK: install-references binds the exclusion count, source and engine.""" + board = MOCK_upgraded_board + _record_installed( + board.stage, lambda r: r["exclusions_installed"].update({field: value}) + ) + with pytest.raises(SystemExit, match="stage.json does not record"): + release.verify_installed_exclusions( + board.stage, board.receipt, board.upgrade.exclusions_text, board.upgrade + ) + + +def test_MOCK_explanations_are_the_builds_bytes(MOCK_upgraded_board): + """MOCK: unchanged baseline explanations fail even when their hash is rebound.""" + board = MOCK_upgraded_board + annotations = board.stage / BUNDLE / "annotations" + release.verify_explanations(annotations, board.upgrade, stage=board.stage) + (annotations / EXPLANATIONS).write_bytes(base_blob(ANNOTATIONS_DIR / EXPLANATIONS)) + with pytest.raises( + SystemExit, match=f"Staged {EXPLANATIONS}.*installed engine upgrade" + ): + release.verify_explanations(annotations, board.upgrade, stage=board.stage) + + +@pytest.mark.parametrize("defect", ["missing", "edited"]) +def test_MOCK_explanation_build_must_exist_and_match_its_pin( + MOCK_upgraded_board, defect +): + """MOCK: inferring stage from the annotation path still checks retained bytes.""" + board = MOCK_upgraded_board + annotations = board.stage / BUNDLE / "annotations" + release.verify_explanations(annotations, board.upgrade) + built = board.stage / driver.BUILD_COPY / EXPLANATIONS + if defect == "missing": + built.unlink() + else: + built.write_bytes(built.read_bytes() + b" ") + with pytest.raises(SystemExit, match="installed build's.*changed"): + release.verify_explanations(annotations, board.upgrade) + + +@pytest.mark.parametrize("defect", ["record_bytes", "receipt_pin"]) +def test_MOCK_installed_exclusion_gate_checks_its_bytes_and_export_binding( + MOCK_upgraded_board, defect +): + """MOCK: the exclusion file is strictly the installed build's recorded bytes.""" + board = MOCK_upgraded_board + if defect == "record_bytes": + path = board.stage / BUNDLE / "us" / EXCLUSIONS + path.write_bytes(path.read_bytes() + b" ") + else: + board.receipt["files"][str(BUNDLE / "us" / EXCLUSIONS)] = "0" * 64 + with pytest.raises( + SystemExit, match="not the installed engine upgrade|does not bind the installed" + ): + release.verify_installed_exclusions( + board.stage, board.receipt, board.upgrade.exclusions_text, board.upgrade + ) + + +def test_MOCK_upgrade_without_regenerated_base_records_requires_an_empty_drop_log( + MOCK_upgraded_board, tmp_path +): + """MOCK: no regenerated base decision still requires an explicit empty log.""" + board = MOCK_upgraded_board + build = mock_builds.real_build( + tmp_path / "MOCK-no-base-drops", regenerated={}, added={} + ) + upgrade = _install_MOCK_build(board.stage, board.receipt, build) + assert not upgrade.regenerated_base + assert release.verify_upgrade(board.stage, board.receipt) == upgrade + _record_installed(board.stage, lambda record: record.pop("adjudications_dropped")) + with pytest.raises(SystemExit, match="records no dropped adjudications"): + release.verify_upgrade(board.stage, board.receipt) + + +def test_MOCK_payload_counts_and_exclusion_order_follow_the_build(MOCK_upgraded_board): + """MOCK: exactly 53 ordered exclusions leave 1,931 outputs for every model.""" + upgrade = MOCK_upgraded_board.upgrade + records = json.loads(upgrade.exclusions_text)["exclusions"] + country = _payload_country(records, upgrade.scored_outputs) + release.verify_scored_outputs(country, records, upgrade) + country["modelStats"][0]["n"] = driver.RELEASE_SCORED + with pytest.raises(SystemExit, match="scored on 1931"): + release.verify_scored_outputs(country, records, upgrade) + country["modelStats"][0]["n"] = upgrade.scored_outputs + country["referenceExclusions"].reverse() + with pytest.raises(SystemExit, match="their order"): + release.verify_scored_outputs(country, records, upgrade) + with pytest.raises(SystemExit, match="exclusion record does not count"): + release.verify_scored_outputs( + _payload_country(records[:-1], upgrade.scored_outputs), + records[:-1], + upgrade, + ) + + +@pytest.mark.parametrize("name", driver.REFERENCE_FILES) +@pytest.mark.parametrize("location", ["stage", "committed", "manifest"]) +def test_MOCK_reference_gates_keep_every_stage_committed_and_manifest_pin( + MOCK_upgraded_board, name, location +): + """MOCK: all five pins are checked, including unchanged scenario sidecars.""" + board = MOCK_upgraded_board + source = board.stage / BUNDLE / "us" + frozen = release.ROOT / FROZEN_RUN + manifest = json.loads(base_blob(SNAPSHOT_DIR / "manifest.json")) + pin = board.upgrade.sha256[EXCLUSIONS] + release.verify_references(source, frozen, manifest, pin, board.upgrade) + if location == "manifest": + manifest["source_run_artifacts"][RUN]["files"][name] = "0" * 64 + else: + path = (source if location == "stage" else frozen) / name + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(SystemExit, match=f"{name}.*changed|Manifest pin for {name}"): + release.verify_references(source, frozen, manifest, pin, board.upgrade) + + +@pytest.mark.parametrize( + "path", + [ + ("committed_snapshot_artifacts", "us_reference_outputs.csv"), + ("audit_annotation_artifacts", "files", EXPLANATIONS), + ], +) +def test_MOCK_reference_gate_checks_additional_committed_and_explanation_pins( + MOCK_upgraded_board, path +): + """MOCK: a previous freeze's extra reference and explanation pins must match.""" + board = MOCK_upgraded_board + manifest = json.loads(base_blob(SNAPSHOT_DIR / "manifest.json")) + _set(manifest, path, "0" * 64) + with pytest.raises(SystemExit, match="Manifest pin for"): + release.verify_references( + board.stage / BUNDLE / "us", + release.ROOT / FROZEN_RUN, + manifest, + board.upgrade.sha256[EXCLUSIONS], + board.upgrade, + ) + + +def test_MOCK_adjudication_gate_accepts_only_kept_ruled_and_recorded_drops( + MOCK_upgraded_board, +): + """MOCK: regenerated ruled cells have no decision and WI_042's is dropped.""" + board = MOCK_upgraded_board + assert ( + release.verify_adjudication_record( + board.stage / BUNDLE / "annotations" / ADJUDICATIONS, + board.stage / BUNDLE / "us", + frozenset(), + [], + board.stage / "audit/cases", + board.upgrade, + ) + == 0 + ) + + +@pytest.mark.parametrize( + "defect", + ["dropped_restored", "kept_dropped", "unruled_rewritten", "extra_decision"], +) +def test_MOCK_adjudication_gate_refuses_any_other_decision_change( + MOCK_upgraded_board, defect +): + """MOCK: an upgrade permits only the driver's exact decisions and drops.""" + board = MOCK_upgraded_board + path = board.stage / BUNDLE / "annotations" / ADJUDICATIONS + record = json.loads(path.read_text()) + if defect == "dropped_restored": + prepared = json.loads((board.stage / "stage.json").read_text()) + record["adjudications"].append(prepared["adjudications_dropped"][0]["entry"]) + elif defect == "kept_dropped": + record["adjudications"].pop() + elif defect == "unruled_rewritten": + record["adjudications"][0]["reasoning"] += " MOCK extra reasoning." + else: + entry = copy.deepcopy(record["adjudications"][0]) + entry["scenario_id"] = "MOCK_extra" + record["adjudications"].append(entry) + path.write_text(driver.record_text(record)) + with pytest.raises(SystemExit): + release.verify_adjudication_record( + path, + board.stage / BUNDLE / "us", + frozenset(), + [], + board.stage / "audit/cases", + board.upgrade, + ) + + +def test_MOCK_added_exclusions_freeze_only_as_the_spec_decides_them( + MOCK_upgraded_board, spec, tmp_path +): + """MOCK: an upgrade's new exclusions (two county outputs) pass when the + spec's upgrade_adjudications decide exactly them, and are refused when it + decides none, as the committed spec does.""" + board = MOCK_upgraded_board + made = mock_builds.real_build( + tmp_path / "MOCK-added", + regenerated={**mock_builds.MOCK_REGENERATED, mock_builds.WI_042: 0.0}, + ) + _install_MOCK_build(board.stage, board.receipt, made) + with pytest.raises(SystemExit, match="must decide exactly"): + release.verify_upgrade(board.stage, board.receipt) + assert release_spec()["upgrade_adjudications"] == [] + spec["upgrade_adjudications"] = copy.deepcopy(mock_builds.MOCK_UPGRADE_ITEMS) + spec["upgrade_adjudications_written_on"] = mock_builds.MOCK_DATE + assert {driver.spec_key(i) for i in spec["upgrade_adjudications"]} == set( + mock_builds.MOCK_NEW + ) + upgrade = release.verify_upgrade(board.stage, board.receipt) + assert upgrade.added == frozenset(mock_builds.MOCK_NEW) + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_MOCK_upgraded_stage_passes_freeze(MOCK_upgraded_board, dry_run): + """MOCK: upgraded references, narratives, counts and pins pass the freeze.""" + board = MOCK_upgraded_board + before = workspace_files() + release.main(["--stage-dir", str(board.stage)] + ["--dry-run"] * dry_run) + if dry_run: + assert workspace_files() == before and board.calls == [] + else: + manifest = json.loads( + (release.ROOT / SNAPSHOT_DIR / "manifest.json").read_text() + ) + assert manifest["reference_exclusions"]["outputs"] == board.upgrade.records + assert ( + manifest["reference_exclusions"]["scored_outputs_per_model"] + == board.upgrade.scored_outputs + ) + pins = driver.upgraded_reference_pins(board.upgrade) + assert all( + sha(release.ROOT / FROZEN_RUN / name) == pin for name, pin in pins.items() + ) + assert ( + manifest["reference_output_refresh"]["policyengine_us_version"] == "2.40.9" + ) + assert ( + manifest["audit_annotation_artifacts"]["files"][EXPLANATIONS] + == board.upgrade.sha256[EXPLANATIONS] + ) + + +def test_MOCK_upgraded_freeze_can_run_twice_with_identical_files(MOCK_upgraded_board): + """MOCK: a repeated freeze accepts the installed build's committed pins.""" + release.main(["--stage-dir", str(MOCK_upgraded_board.stage)]) + before = workspace_files() + release.main(["--stage-dir", str(MOCK_upgraded_board.stage)]) + assert workspace_files() == before + + +@pytest.mark.parametrize( + "defect", + [ + "csv_bytes", + "top_csv_bytes", + "meta_bytes", + "exclusions_bytes", + "explanations_bytes", + "scenario_bytes", + "csv_pin", + "meta_pin", + "explanations_pin", + "committed_csv_pin", + "exclusions_pin", + "exclusion_count", + "scored_count", + "refresh_engine", + "refresh_date", + "refresh_csv_pin", + "unlisted_manifest", + *[ + f"refresh:{field}" + for field in ( + "regenerated_at_utc", + "policyengine_version", + "policyengine_us_data_build_id", + "policyengine_us_dataset", + "policyengine_us_dataset_uri", + "policyengine_us_data_artifact_sha256", + ) + ], + ], +) +def test_MOCK_verify_frozen_refuses_incorrect_build_files_pins_and_counts( + MOCK_upgraded_board, defect +): + """MOCK: allowed manifest changes must still state the exact installed build.""" + board = MOCK_upgraded_board + release.main(["--stage-dir", str(board.stage)]) + snapshot = release.ROOT / SNAPSHOT_DIR + annotations = release.ROOT / ANNOTATIONS_DIR + files = ("source_run_artifacts", RUN, "files") + paths = { + "csv_pin": (*files, "reference_outputs.csv"), + "meta_pin": (*files, "reference_outputs.csv.meta.json"), + "explanations_pin": ("audit_annotation_artifacts", "files", EXPLANATIONS), + "committed_csv_pin": ( + "committed_snapshot_artifacts", + "us_reference_outputs.csv", + ), + "exclusions_pin": (*files, EXCLUSIONS), + "exclusion_count": ("reference_exclusions", "outputs"), + "scored_count": ("reference_exclusions", "scored_outputs_per_model"), + "refresh_engine": ("reference_output_refresh", "policyengine_us_version"), + "refresh_date": ("reference_output_refresh", "date"), + "refresh_csv_pin": ("reference_output_refresh", "reference_csv_sha256"), + "unlisted_manifest": ("population_weight_artifact", "sha256"), + **{ + f"refresh:{field}": ("reference_output_refresh", field) + for field in ( + "regenerated_at_utc", + "policyengine_version", + "policyengine_us_data_build_id", + "policyengine_us_dataset", + "policyengine_us_dataset_uri", + "policyengine_us_data_artifact_sha256", + ) + }, + } + byte_paths = { + "csv_bytes": snapshot / "runs" / RUN / "reference_outputs.csv", + "top_csv_bytes": snapshot / "us_reference_outputs.csv", + "meta_bytes": snapshot / "runs" / RUN / "reference_outputs.csv.meta.json", + "exclusions_bytes": snapshot / "runs" / RUN / EXCLUSIONS, + "explanations_bytes": annotations / EXPLANATIONS, + "scenario_bytes": snapshot / "runs" / RUN / "scenarios.csv", + } + if defect in byte_paths: + path = byte_paths[defect] + path.write_bytes(path.read_bytes() + b" ") + else: + _edit_json( + snapshot / "manifest.json", lambda m: _set(m, paths[defect], "MOCK wrong") + ) + serving = base_serving() + with pytest.raises(SystemExit): + release.verify_frozen( + stage=board.stage, + snapshot=snapshot, + annotations=annotations, + exclusions_sha256=board.upgrade.sha256[EXCLUSIONS], + pointer=json.loads((release.ROOT / release.POINTER).read_text()), + window=WINDOW, + snapshot_date="2026-10-09", + previous_serving=serving, + incumbents=set(serving["models"]), + runs=RUNS, + upgrade=board.upgrade, + ) + + +def test_MOCK_upgrade_manifest_permissions_name_only_the_new_paths(MOCK_upgraded_board): + """MOCK: explicitly allowed upgrade leaves move; every other leaf stays pinned.""" + upgrade = MOCK_upgraded_board.upgrade + before = json.loads(_manifest_text()) + allowed = set(release.UPGRADE_MANIFEST_CHANGES) | set( + release.UPGRADE_MANIFEST_UNPINNED + ) + for path in _leaves(): + after = copy.deepcopy(before) + _set(after, path, "MOCK changed") + expected = [] if _allowed(path) or path in allowed else [path] + assert release.manifest_problems(before, after, upgrade) == expected, path + after = copy.deepcopy(before) + after["reference_output_refresh"]["MOCK_unlisted"] = "MOCK" + assert release.manifest_problems(before, after, upgrade) == [ + ("reference_output_refresh", "MOCK_unlisted") + ] + + +def test_MOCK_versions_count_records_on_the_build_engine(MOCK_upgraded_board, tmp_path): + """MOCK: a new engine's retained records appear in the description's counts.""" + made = mock_builds.real_build( + tmp_path / "MOCK-engine-count", + regenerated={**mock_builds.MOCK_REGENERATED, mock_builds.WI_042: 0.0}, + ) + upgrade = mock_builds.load(made) + records = json.loads(upgrade.exclusions_text)["exclusions"] + versions = release.release_versions(head_versions(), records, "2026-10-09", upgrade) + description = _live_version(versions)["description"] + assert description.startswith( + f"Scored reference outputs from {upgrade.engine_version} (" + ) + assert "2 from 2.40.9" in description + assert "were re-reviewed" not in description + assert ( + release.release_versions(versions, records, "2026-10-09", upgrade) == versions + ) + records[-1] = {**records[-1], "engine_version": "MOCK unsupported engine"} + with pytest.raises(SystemExit, match="counts exclusions decided on"): + release.release_versions(head_versions(), records, "2026-10-09", upgrade) + + +def _MOCK_description_upgrade() -> driver.Upgrade: + """MOCK: minimal upgrade data solely for the version description formatter.""" + return driver.Upgrade( + engine_version=mock_builds.MOCK_ENGINE, + previous_engine_version=driver.BASE_ENGINE, + revision={"excluded_outputs_rechecked": []}, + changed={}, + regenerated_base=frozenset(), + regenerated_ruled=frozenset(), + added=frozenset(), + records=driver.RELEASE_EXCLUSIONS, + exclusions_text=release_exclusions_text(), + sha256={}, + ) + + +@pytest.fixture +def MOCK_refresh_sidecar(tmp_path): + """MOCK: a retained sidecar for timestamp validation alone, without an audit.""" + stage = tmp_path / "MOCK-stage" + retained = stage / driver.BUILD_COPY + retained.mkdir(parents=True) + meta = json.loads(base_blob(FROZEN_RUN / driver.META_NAME)) + meta["policyengine_bundles"]["us"]["model_version"] = "2.40.9" + path = retained / driver.META_NAME + upgrade = replace(_MOCK_description_upgrade(), sha256={driver.CSV_NAME: "f" * 64}) + return types.SimpleNamespace(stage=stage, path=path, meta=meta, upgrade=upgrade) + + +@pytest.mark.parametrize( + "defect, timestamp", + [ + ("missing", None), + ("none", None), + ("integer", 20261009), + ("boolean", True), + ("empty", ""), + ("whitespace", " "), + ("date_only", "2026-10-09"), + ("naive", "2026-10-09T18:00:00"), + ("non_utc", "2026-10-09T18:00:00+01:00"), + ("invalid_date", "2026-10-32T18:00:00+00:00"), + ("invalid", "MOCK bad timestamp"), + ("compact_date", "20261009T18:00:00Z"), + ], +) +def test_MOCK_refresh_timestamp_refuses_missing_or_non_UTC_ISO_values( + MOCK_refresh_sidecar, defect, timestamp +): + """MOCK: malformed timestamps cannot determine a frozen regeneration date.""" + sidecar = MOCK_refresh_sidecar + if defect == "missing": + sidecar.meta.pop("regenerated_at_utc", None) + else: + sidecar.meta["regenerated_at_utc"] = timestamp + sidecar.path.write_text(json.dumps(sidecar.meta)) + with pytest.raises(SystemExit, match="nonempty ISO timestamp in UTC"): + release.upgraded_reference_refresh(sidecar.stage, sidecar.upgrade) + + +@pytest.mark.parametrize( + "timestamp", ["2026-10-09T18:00:00Z", "2026-10-09T18:00:00+00:00"] +) +def test_MOCK_refresh_timestamp_preserves_valid_UTC_spellings( + MOCK_refresh_sidecar, timestamp +): + """MOCK: both explicit UTC offsets remain byte-for-byte as the sidecar states.""" + sidecar = MOCK_refresh_sidecar + sidecar.meta["regenerated_at_utc"] = timestamp + sidecar.path.write_text(json.dumps(sidecar.meta)) + refresh = release.upgraded_reference_refresh(sidecar.stage, sidecar.upgrade) + assert refresh["date"] == "2026-10-09" + assert refresh["regenerated_at_utc"] == timestamp + assert refresh["reference_csv_sha256"] == sidecar.upgrade.sha256[driver.CSV_NAME] + + +@examples(25) +@given(count=st.integers(min_value=0, max_value=58)) +def test_MOCK_version_rechecks_count_and_engine_are_truthful_and_idempotent(count): + """MOCK: every recheck count updates the clause and keeps repeated output exact.""" + upgrade = _MOCK_description_upgrade() + upgrade = replace( + upgrade, + revision={"excluded_outputs_rechecked": [{"MOCK": i} for i in range(count)]}, + ) + versions = release.release_versions( + head_versions(), release_exclusions(), "2026-10-09", upgrade + ) + description = _live_version(versions)["description"] + assert description.startswith( + f"Scored reference outputs from {upgrade.engine_version} (" + ) + assert "move on 2.38.6 were re-reviewed" not in description + if count: + assert ( + f"and the {count} that move on 2.40.9 were re-reviewed and stay excluded" + in description + ) + else: + assert "were re-reviewed" not in description + assert ( + release.release_versions(versions, release_exclusions(), "2026-10-09", upgrade) + == versions + ) + + +def test_MOCK_omitted_recheck_list_has_the_same_description_as_zero_rechecks(): + """MOCK: the driver permits an omitted empty recheck claim in the sidecar.""" + upgrade = _MOCK_description_upgrade() + records = release_exclusions() + expected = release.release_versions(head_versions(), records, "2026-10-09", upgrade) + omitted = replace(upgrade, revision={}) + assert ( + release.release_versions(head_versions(), records, "2026-10-09", omitted) + == expected + ) + + +@pytest.mark.parametrize( + "prefix", + [ + "MOCK reference outputs from", + "Scored reference outputs from policyengine-us 9.9", + ], +) +def test_MOCK_versions_refuse_an_unexpected_scored_reference_prefix(prefix): + """MOCK: an unrelated engine description cannot be silently rewritten.""" + versions = head_versions() + live = _live_version(versions) + live["description"] = live["description"].replace( + "Scored reference outputs from policyengine-us 2.38.6", prefix, 1 + ) + with pytest.raises(SystemExit, match="unexpected reference engine"): + release.release_versions( + versions, release_exclusions(), "2026-10-09", _MOCK_description_upgrade() + ) + + +@pytest.mark.parametrize( + "defect", ["missing_anchor", "unknown_engine", "malformed_clause"] +) +def test_MOCK_versions_refuse_an_unexpected_recheck_description_shape(defect): + """MOCK: an unrecognized re-review clause must never survive into a release.""" + versions = head_versions() + live = _live_version(versions) + old, new = { + "missing_anchor": ("); same household facts", "); MOCK different facts"), + "unknown_engine": ( + "move on 2.38.6 were re-reviewed", + "move on 9.9 were re-reviewed", + ), + "malformed_clause": ( + "were re-reviewed and stay excluded", + "MOCK unreviewed and stay excluded", + ), + }[defect] + live["description"] = live["description"].replace(old, new, 1) + with pytest.raises(SystemExit, match="unexpected re-review shape"): + release.release_versions( + versions, release_exclusions(), "2026-10-09", _MOCK_description_upgrade() + ) + + +@examples(5) +@given( + day=st.dates( + min_value=datetime.date(2026, 1, 1), max_value=datetime.date(2030, 12, 31) + ) +) +def test_without_an_upgrade_optional_arguments_preserve_existing_freeze_behavior( + frozen_board, spec, day +): + """Absent and explicit None upgrades freeze identical files for any date.""" + board = frozen_board + assert ( + release.verify_upgrade( + board.stage, json.loads((board.stage / "release-ready.json").read_text()) + ) + is None + ) + records = release_exclusions() + country = _payload_country(records) + assert release.verify_scored_outputs( + country, records + ) == release.verify_scored_outputs(country, records, None) + assert release.release_versions( + head_versions(), records, day.isoformat() + ) == release.release_versions(head_versions(), records, day.isoformat(), None) + before = json.loads(_manifest_text()) + assert release.manifest_problems(before, before) == release.manifest_problems( + before, before, None + ) + spec["snapshot_date"] = day.isoformat() + + def published_files(): + relative = board.stage.relative_to(release.ROOT) + return { + path: blob + for path, blob in workspace_files().items() + if not path.is_relative_to(relative) + } + + _record_installed( + board.stage, lambda record: record.pop("references_installed", None) + ) + board.rebind(restamp_stage=False) + release.main(["--stage-dir", str(board.stage)]) + absent = published_files() + _record_installed( + board.stage, lambda record: record.update(references_installed=None) + ) + board.rebind(restamp_stage=False) + release.main(["--stage-dir", str(board.stage)]) + assert published_files() == absent + + +@pytest.mark.parametrize( + "stage", + [ + REPO / "results/local/musewonder/stage", + REPO / "results/local/upgrade-scratch/install-references", + ], +) +def test_optional_live_stage_upgrade_bindings_are_read_only(stage): + """Read-only optional stages, including the MOCK install-references rehearsal.""" + path = stage / "stage.json" + if not path.is_file(): + pytest.skip("no live stage in this checkout") + before = {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} + prepared = json.loads(path.read_text()) + if prepared.get("references_installed"): + driver.verify_installed_references(stage, prepared) + else: + assert driver.stage_upgrade(stage) is None + assert {p: p.read_bytes() for p in stage.rglob("*") if p.is_file()} == before + + +# --- The version list -------------------------------------------------------------- + + +def head_versions() -> dict: + return json.loads(base_blob(Path(release.VERSIONS))) + + +def test_the_version_description_counts_the_releases_exclusions(): + versions = head_versions() + out = release.release_versions(versions, release_exclusions(), "2026-10-09") + live, before = _live_version(out), _live_version(versions) + # The record is release 20261010's, so only the model count moves. + assert live["description"] == ( + before["description"].replace(" - 47 models", " - 48 models") + ) + assert live["snapshotLabel"] == "Snapshot 2026-10-09" + assert { + k: v for k, v in live.items() if k not in ("description", "snapshotLabel") + } == {k: v for k, v in before.items() if k not in ("description", "snapshotLabel")} + + +@examples(100) +@given( + day=st.dates( + min_value=datetime.date(2026, 1, 1), max_value=datetime.date(2030, 12, 31) + ), + drop=st.integers(min_value=0, max_value=57), +) +def test_release_versions_changes_nothing_when_applied_twice(day, drop): + """For any snapshot date and any record of both engines, the second + application changes nothing, and only the live version's description and + label differ from the input.""" + records = release_exclusions() + records = records[:drop] + records[drop + 1 :] + versions = head_versions() + once = release.release_versions(versions, records, day.isoformat()) + assert release.release_versions(once, records, day.isoformat()) == once + assert versions == head_versions() + for got, had in zip(once["versions"], versions["versions"], strict=True): + if had["id"] == versions["default"]: + assert got["snapshotLabel"] == f"Snapshot {day.isoformat()}" + assert got["description"].endswith(" - 48 models") + assert f"the {len(records)} excluded outputs" in got["description"] + got = {**got, "description": None, "snapshotLabel": None} + had = {**had, "description": None, "snapshotLabel": None} + assert got == had + + +@pytest.mark.parametrize("defect", ["default", "pointer", "engine", "shape"]) +def test_release_versions_refuses_an_unexpected_version_list(defect): + versions = head_versions() + records = release_exclusions() + live = _live_version(versions) + message = "The default version is not the one live-pointer version" + if defect == "default": + versions["default"] = "9.9" + elif defect == "pointer": + live["artifact"] = {"tag": "dashboard-data-20261010"} + elif defect == "engine": + records = [*records, {**records[0], "engine_version": "policyengine-us 3"}] + message = "counts exclusions decided on" + else: + live["description"] = live["description"].replace(" - 47 models", "") + message = "unexpected model-count shape" + with pytest.raises(SystemExit, match=message): + release.release_versions(versions, records, "2026-10-09") + + +def test_the_pointer_names_the_release_asset(tmp_path): + payload = tmp_path / "data-board48.json" + payload.write_text('{"countries": {}}') + tag = driver.RELEASE_TAG + pointer = release.release_pointer(tag, payload, "a" * 64) + assert pointer == { + "version": 1, + "repo": "PolicyEngine/policybench", + "tag": tag, + "asset": "dashboard-data.json", + "url": ( + "https://github.com/PolicyEngine/policybench/releases/download/" + f"{tag}/dashboard-data.json" + ), + "sha256": "a" * 64, + "bytes": len('{"countries": {}}'), + } + + +# --- Serving ----------------------------------------------------------------------- + + +def base_serving() -> dict: + return json.loads(base_blob(SNAPSHOT_DIR / "model_serving_config.json")) + + +def _freezers_serving(tmp_path) -> dict: + path = tmp_path / "serving.json" + _serving_from_the_registry(path) + return json.loads(path.read_text()) + + +def test_merge_serving_keeps_each_incumbent_row_and_names_the_run(tmp_path): + previous = base_serving() + incumbents = set(previous["models"]) + merged = release.merge_serving( + _freezers_serving(tmp_path), previous, incumbents, RUNS + ) + assert release.serving_problems(merged, previous, incumbents, RUNS) == [] + assert {m: merged["models"][m] for m in incumbents} == previous["models"] + assert merged["models"][NEW]["evidence"]["run"] == RUNS[SLUG] + + +@examples(100) +@given( + pick=st.integers(min_value=0, max_value=45), + field=st.sampled_from(["request_timeout_seconds", "evidence", "x_new"]), + value=JSON_VALUES, +) +def test_an_incumbent_serving_row_that_changes_is_refused(tmp_path, pick, field, value): + """Whatever the freezer wrote for an incumbent, the merge restores release + 20261010's row; and any change to a merged incumbent row is reported.""" + previous = base_serving() + incumbents = set(previous["models"]) + model = sorted(incumbents)[pick] + serving = _freezers_serving(tmp_path) + serving["models"][model][field] = value + merged = release.merge_serving(serving, previous, incumbents, RUNS) + assert merged["models"][model] == previous["models"][model] + edited = copy.deepcopy(merged) + edited["models"][model] = {**merged["models"][model], "x_edit": [value]} + problems = release.serving_problems(edited, previous, incumbents, RUNS) + assert problems == [f"{model} is not release 20261010's row"] + + +@pytest.mark.parametrize("defect", ["registry", "other_run", "summary", "roster"]) +def test_the_new_rows_serving_evidence_must_name_its_run(tmp_path, defect): + previous = base_serving() + incumbents = set(previous["models"]) + merged = release.merge_serving( + _freezers_serving(tmp_path), previous, incumbents, RUNS + ) + if defect == "registry": + merged["models"][NEW]["evidence"] = {"kind": "registry"} + message = "has no run-state evidence" + elif defect == "other_run": + merged["models"][NEW]["evidence"]["run"] = "adds202609/musewonder" + message = "evidence names run 'adds202609/musewonder'" + elif defect == "summary": + merged["evidence_summary"]["run_state"] += 1 + message = "is not the rows' tally" + else: + del merged["models"][INCUMBENT] + message = "roster differs" + problems = release.serving_problems(merged, previous, incumbents, RUNS) + assert any(message in problem for problem in problems), problems + with pytest.raises(SystemExit, match="Missing frozen run-state evidence"): + serving = _freezers_serving(tmp_path) + serving["models"][NEW]["evidence"] = {"kind": "registry"} + release.merge_serving(serving, previous, incumbents, RUNS) + + +def test_the_new_models_evidence_names_the_pinned_run(): + assert release.evidence_runs() == RUNS + + +@pytest.mark.parametrize( + "run", + [None, "musewonder/run", "musewonder-runs/muse/run", "a/musewonder/run/x"], +) +def test_evidence_runs_refuses_pins_that_name_no_supervised_run(monkeypatch, run): + record = {"inputs": {SLUG: {"run": run}}} + monkeypatch.setattr(driver, "head_blob", lambda path: json.dumps(record).encode()) + with pytest.raises(SystemExit, match="does not name the supervised run"): + release.evidence_runs() + + +def test_the_freeze_reads_20261010_predictions_and_serving_from_git(monkeypatch): + import gzip + + blobs = {} + + def blob(path): + blobs[str(path)] = True + if path.name == "predictions.csv.gz": + return gzip.compress(b"model,scenario_id,variable,prediction\nm,s,v,1\n") + return json.dumps({"models": {"m": {}}}).encode() + + monkeypatch.setattr(driver, "base_commit_blob", blob) + assert list(release.base_prediction_rows().model) == ["m"] + assert release.base_serving_config() == {"models": {"m": {}}} + assert blobs == { + f"paper/snapshot/20260501/runs/{RUN}/predictions.csv.gz": True, + "paper/snapshot/20260501/model_serving_config.json": True, + } + + +# --- The freezer's destinations ------------------------------------------------ + + +def test_the_freezer_must_write_into_this_checkout(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + point_freeze_at(workspace, monkeypatch) + release.verify_freezer_destinations(freezer) + monkeypatch.setattr(freezer, "RUN_DEST", tmp_path / "elsewhere") + with pytest.raises(SystemExit, match="would write outside this checkout"): + release.verify_freezer_destinations(freezer) + monkeypatch.setattr(freezer, "RUN_DEST", workspace / FROZEN_RUN) + monkeypatch.setattr(freezer, "RUN_LABEL", "another_run") + with pytest.raises(SystemExit, match="another_run"): + release.verify_freezer_destinations(freezer) + + +def test_the_freezer_is_this_checkouts(): + release.verify_freezer_destinations(freezer) + + +def test_the_stage_must_be_inside_this_checkout(tmp_path, monkeypatch): + point_freeze_at(tmp_path / "workspace", monkeypatch) + with pytest.raises(SystemExit): + release.main(["--stage-dir", str(tmp_path / "stage"), "--dry-run"]) + + +# --- References -------------------------------------------------------------------- + + +@pytest.fixture +def references(tmp_path): + """The staged references (the release's exclusion record), the committed + copies and the manifest release 20261010 committed, from git.""" + staged = pinned_references(tmp_path / "staged") + frozen = base_references(tmp_path / "frozen") + manifest = json.loads(base_blob(SNAPSHOT_DIR / "manifest.json")) + return staged, frozen, manifest, sha(staged / EXCLUSIONS) + + +def test_unchanged_references_pass(references): + release.verify_references(*references) + + +def test_a_second_freeze_finds_this_releases_exclusion_record(references): + staged, frozen, manifest, exclusions = references + shutil.copyfile(staged / EXCLUSIONS, frozen / EXCLUSIONS) + manifest["source_run_artifacts"][RUN]["files"][EXCLUSIONS] = exclusions + release.verify_references(staged, frozen, manifest, exclusions) + + +@pytest.mark.parametrize("name", release.PINNED_REFERENCES) +@pytest.mark.parametrize("where", ["Staged", "Committed", "Manifest pin"]) +def test_any_reference_value_revision_is_refused(references, name, where): + staged, frozen, manifest, exclusions = references + if where == "Staged": + (staged / name).write_bytes((staged / name).read_bytes() + b"\n") + elif where == "Committed": + (frozen / name).write_bytes((frozen / name).read_bytes() + b"\n") + else: + manifest["source_run_artifacts"][RUN]["files"][name] = "0" * 64 + with pytest.raises(SystemExit, match=f"{where} (for )?{name}"): + release.verify_references(staged, frozen, manifest, exclusions) + + +@pytest.mark.parametrize("where", ["Staged", "Committed", "Manifest pin"]) +def test_the_exclusion_record_is_release_20261010s_or_this_releases(references, where): + staged, frozen, manifest, exclusions = references + edited = release_exclusions_text().replace('"decided_by"', '"decided_by" ', 1) + if where == "Staged": + (staged / EXCLUSIONS).write_text(edited) + message = "Staged reference_exclusions.json changed: it is not the release" + elif where == "Committed": + (frozen / EXCLUSIONS).write_text(edited) + message = "Committed reference_exclusions.json changed: it is neither" + else: + manifest["source_run_artifacts"][RUN]["files"][EXCLUSIONS] = "0" * 64 + message = "Manifest pin for reference_exclusions.json is neither" + with pytest.raises(SystemExit, match=message): + release.verify_references(staged, frozen, manifest, exclusions) + + +def test_the_staged_explanations_must_be_release_20261010s(tmp_path): + annotations = tmp_path / "annotations" + annotations.mkdir() + path = annotations / EXPLANATIONS + path.write_bytes(base_blob(ANNOTATIONS_DIR / EXPLANATIONS)) + release.verify_explanations(annotations) + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(SystemExit, match="rewords no reference explanation"): + release.verify_explanations(annotations) + path.unlink() + with pytest.raises(SystemExit, match="rewords no reference explanation"): + release.verify_explanations(annotations) + + +# --- Adjudications ------------------------------------------------------------------- + + +@pytest.fixture +def ruled(tmp_path): + """The record adjudicate-exclusions writes on release 20261010's (from git + at BASE_COMMIT), its new ruled cases' bound verdicts, and the release's + references: the baseline every staged record is checked against.""" + cases = tmp_path / "cases" + write_ruled_verdicts(cases) + record = ruled_record(cases) + staged = tmp_path / ADJUDICATIONS + staged.write_text(driver.record_text(record)) + source_us = pinned_references(tmp_path / "us") + return types.SimpleNamespace( + staged=staged, record=record, cases=cases, source_us=source_us + ) + + +def _write(path, record): + path.write_text(driver.record_text(record)) + + +def _case(entry): + return f"{entry['country']}__{entry['scenario_id']}__{entry['variable']}" + + +def _ruled_cases() -> set[str]: + return { + case_dir_name(*driver.spec_key(item)) + for item in release_spec()["adjudications"] + } + + +def _unruled(record: dict) -> list[dict]: + return [e for e in record["adjudications"] if _case(e) not in _ruled_cases()] + + +def _verify(ruled, rejudged=frozenset(), amendments=()): + return release.verify_adjudication_record( + ruled.staged, + ruled.source_us, + frozenset(rejudged), + list(amendments), + ruled.cases, + ) + + +def test_the_staged_record_passes_the_record_gate(ruled): + """The spec rules nothing: the record adjudicate-exclusions writes is + release 20261010's, byte for byte, and it excludes the release's 58.""" + assert ruled.staged.read_bytes() == base_blob(ADJUDICATIONS_PATH) + assert _verify(ruled) == 0 + from policybench.adjudications import excluded_case_keys, load_adjudications + + assert len(excluded_case_keys(load_adjudications(ruled.staged))) == 58 + + +def test_the_record_must_exclude_what_the_staged_record_lists(ruled): + """With an exclusion record staged that lists one output fewer, the + recorded decisions exclude an output the record does not.""" + doc = json.loads(release_exclusions_text()) + doc["exclusions"].pop() + pinned_references(ruled.source_us, exclusions=driver.exclusions_text(doc)) + with pytest.raises(SystemExit, match="disagree on the excluded outputs"): + _verify(ruled) + + +def test_triage_may_add_a_decision_that_keeps_the_output_scored(ruled): + record = ruled.record + scored = next( + e + for e in _unruled(record) + if not e.get("excluded_from_scoring") and e["judge_model"] == driver.JUDGE_MODEL + ) + added = {**copy.deepcopy(scored), "scenario_id": "scenario_999"} + record["adjudications"].append(added) + _write(ruled.staged, record) + # A new decision names the case's current bound Opus 5.5 verdict. + with pytest.raises(SystemExit, match="not restated by the restate script"): + _verify(ruled, rejudged={_case(added)}) + _bind_current_verdict(ruled.cases, added) + assert _verify(ruled, rejudged={_case(added)}) == 1 + # A decision on a case Muse Wonder did not re-open has no reason to + # appear. + with pytest.raises(SystemExit, match="did not re-open"): + _verify(ruled) + + +def _restated(entry, day="2026-10-08"): + from restate_gpt61sol_adjudications import named_item + + return { + **entry, + "judge_failure_subtype": "thresholds_rates", + "judge_rejudged_on": day, + "judge_previous": [*entry.get("judge_previous", []), named_item(entry)], + **({"judged_on_utc": day} if "judged_on_utc" in entry else {}), + } + + +def _bind_current_verdict(cases: Path, entry: dict) -> None: + case = cases / _case(entry) + case.mkdir(parents=True, exist_ok=True) + verdict = case / "verdict.json" + verdict.write_text('{"case_failure_source": "llm_error"}') + (case / "verdict.meta.json").write_text( + json.dumps( + { + "verdict_sha256": sha(verdict), + "judge_model_requested": driver.JUDGE_MODEL, + "judge_model_reported": [driver.JUDGE_MODEL], + "judged_at_utc": "2026-10-08T05:10:00+00:00", + } + ) + ) + + +def test_a_rejudged_case_may_be_restated(ruled): + record = ruled.record + entry = next(e for e in _unruled(record) if e["judge_model"] == driver.JUDGE_MODEL) + index = record["adjudications"].index(entry) + record["adjudications"][index] = _restated(entry) + _bind_current_verdict(ruled.cases, entry) + _write(ruled.staged, record) + assert _verify(ruled, rejudged={_case(entry)}) == 0 + # Not re-opened, the same restatement is a rewrite. + with pytest.raises(SystemExit, match="change recorded decisions"): + _verify(ruled) + + +@pytest.mark.parametrize( + "edit", + ["judge_class", "decision_class", "reasoning", "decision_order", "entry_order"], +) +def test_a_rewrite_of_an_incumbent_only_case_is_refused(ruled, edit): + """No field of a case Muse Wonder did not re-open may change, key + order and entry order included; a re-opened case may change its judge + fields only.""" + record = ruled.record + entries = record["adjudications"] + first, second = _unruled(record)[:2] + i, j = entries.index(first), entries.index(second) + rejudged = set() + if edit == "judge_class": + first["judge_failure_subtype"] = "thresholds_rates" + elif edit == "decision_class": + first["adjudicated_failure_subtype"] = "other" + rejudged = {_case(first)} + elif edit == "reasoning": + first["reasoning"] += " Restated." + rejudged = {_case(first)} + elif edit == "decision_order": + entries[i] = dict(reversed(list(first.items()))) + rejudged = {_case(first)} + else: + entries[i], entries[j] = entries[j], entries[i] + _write(ruled.staged, record) + with pytest.raises(SystemExit, match="change recorded decisions|entry order"): + _verify(ruled, rejudged=rejudged) + + +def test_a_dropped_decision_is_refused(ruled): + record = ruled.record + record["adjudications"].remove(_unruled(record)[-1]) + _write(ruled.staged, record) + with pytest.raises(SystemExit, match="drop recorded decisions"): + _verify(ruled) + + +def test_a_new_exclusion_is_refused(ruled): + record = ruled.record + excluded = next(e for e in _unruled(record) if e.get("excluded_from_scoring")) + added = { + **copy.deepcopy(excluded), + "scenario_id": "scenario_999", + "judge_model": driver.JUDGE_MODEL, + } + record["adjudications"].append(added) + _write(ruled.staged, record) + _bind_current_verdict(ruled.cases, added) + with pytest.raises(SystemExit, match="disagree on the excluded outputs"): + _verify(ruled, rejudged={_case(added)}) + + +def test_the_committed_base_record_is_in_the_committed_form(): + text = base_blob(ADJUDICATIONS_PATH).decode() + assert text == driver.record_text(json.loads(text)) + + +@pytest.mark.parametrize( + "defect", + ["duplicate_key", "note", "conventions", "compact", "named_wave", "wave_twice"], +) +def test_bytes_the_entry_gate_cannot_see_are_refused(ruled, spec, defect): + """A duplicate key hides text a parser drops; the note and conventions are + outside the entries; a record in another layout could hide either; and + the conventions name no wave this release does not add, and release + 20261010's last wave exactly once.""" + record = ruled.record + text = driver.record_text(record) + base = driver.base_adjudication_record() + if defect == "duplicate_key": + text = text.replace( + '"adjudicator": "developer"', + '"adjudicator": "SMUGGLED", "adjudicator": "developer"', + 1, + ) + elif defect == "note": + record["note"] = "Rewritten." + text = driver.record_text(record) + elif defect == "conventions": + record["date_conventions"] = {} + text = driver.record_text(record) + elif defect == "compact": + text = json.dumps(record) + elif defect == "named_wave": + # An engine upgrade's wave, which this stage did not install. + spec["upgrade_adjudications_written_on"] = mock_builds.MOCK_DATE + record["date_conventions"] = driver.release_date_conventions( + base["date_conventions"], + driver.DecisionWave(mock_builds.MOCK_DATE, excludes=True, affirms=False), + ) + assert record["date_conventions"] != base["date_conventions"] + text = driver.record_text(record) + else: + last = ( + "The 2026-10-10 wave's decisions exclude the outputs the references' " + "engine upgrade moved onto an input the prompt does not state, and " + "were written on 2026-10-10 UTC." + ) + assert base["date_conventions"].count(last) == 1 + record["date_conventions"] = base["date_conventions"].replace( + last, f"{last} {last}" + ) + text = driver.record_text(record) + ruled.staged.write_text(text) + with pytest.raises( + SystemExit, match="committed form|only entries may change|not release 20261010" + ): + _verify(ruled) + + +@pytest.fixture +def restated(tmp_path): + """A re-opened entry restated as the restate script does, with its verdict.""" + base = driver.base_adjudications() + entry = next( + e + for e in base + if e["judge_model"] == driver.JUDGE_MODEL + and e.get("judge_previous") + and _case(e) not in _ruled_cases() + ) + cases = tmp_path / "cases" + _bind_current_verdict(cases, entry) + staged = [_restated(e) if e is entry else e for e in base] + index = next(i for i, e in enumerate(staged) if _case(e) == _case(entry)) + return base, staged, frozenset({_case(entry)}), cases, index + + +def test_a_restated_entry_passes_the_restatement_check(restated): + base, staged, rejudged, cases, _ = restated + driver.verify_restatements(base, staged, rejudged, cases) + + +# Each rewrites the appended item: the verdict the re-judge replaced must stay +# the one 20261010's entry names, as its seed sidecar records it. +APPENDED_ITEM_TAMPERS = { + "appended_judged_on": {"judged_on": "2026-09-28"}, + "appended_class": {"judge_failure_source": "reference_error"}, + "appended_subtype": {"judge_failure_subtype": "other"}, + "appended_flag": {"judge_reference_suspect": "yes"}, + "appended_judge": {"judge_model": "claude-opus-5"}, + "appended_flag_source": {"judge_reference_suspect_source": "a-human-typed-this"}, +} + + +@pytest.mark.parametrize( + "tamper", + [ + "judge_model", + "emptied_history", + "two_items", + "rewritten_history", + "day", + *APPENDED_ITEM_TAMPERS, + ], +) +def test_judge_fields_written_by_hand_are_refused(restated, tamper): + base, staged, rejudged, cases, index = restated + entry = copy.deepcopy(staged[index]) + if tamper == "judge_model": + entry["judge_model"] = "a-human-typed-this" + elif tamper == "emptied_history": + entry["judge_previous"] = [] + elif tamper == "two_items": + entry["judge_previous"] = entry["judge_previous"] + entry["judge_previous"][-1:] + elif tamper == "rewritten_history": + entry["judge_previous"][0] = {**entry["judge_previous"][0], "judged_on": "1999"} + elif tamper in APPENDED_ITEM_TAMPERS: + appended = entry["judge_previous"][-1] + edit = APPENDED_ITEM_TAMPERS[tamper] + assert all(appended.get(k) != v for k, v in edit.items()) + entry["judge_previous"][-1] = {**appended, **edit} + else: + entry["judge_rejudged_on"] = "1999-01-01" + staged[index] = entry + with pytest.raises(SystemExit, match="not restated by the restate script"): + driver.verify_restatements(base, staged, rejudged, cases) + + +@pytest.fixture(scope="module") +def ruled_cases(tmp_path_factory): + cases = tmp_path_factory.mktemp("ruled") / "cases" + write_ruled_verdicts(cases) + return cases + + +@functools.cache +def _gate_baseline(cases: Path) -> tuple[str, str, tuple[int, ...]]: + """Release 20261010's entries and the ruled record's, and the indices of + the ruled record's entries outside the ruled outputs (all of them: the + spec rules none).""" + from policybench.adjudications import parse_adjudications + + staged = parse_adjudications(ruled_record(cases), "ruled") + outside = tuple( + index for index, e in enumerate(staged) if _case(e) not in _ruled_cases() + ) + return json.dumps(driver.base_adjudications()), json.dumps(staged), outside + + +@examples(200) +@given( + pick_entry=st.integers(min_value=0, max_value=10**6), + pick=st.integers(min_value=0, max_value=10**6), + mutation=st.sampled_from( + ["value", "delete", "add_decision_key", "add_judge_key", "swap_decisions"] + ), + reopened=st.booleans(), +) +def test_the_gate_allows_exactly_the_judge_fields_of_reopened_cases( + ruled_cases, pick_entry, pick, mutation, reopened +): + """For every entry outside the ruled ones (all) and every kind of single + change: a change to a judge field passes only where the case is + re-opened; any other change fails.""" + from restate_gpt61sol_adjudications import JUDGE_FIELDS + + base_text, staged_text, outside = _gate_baseline(ruled_cases) + base, staged = json.loads(base_text), json.loads(staged_text) + index = outside[pick_entry % len(outside)] + entry = staged[index] + keys = list(entry) + judge = mutation == "add_judge_key" + if mutation in ("value", "delete"): + key = keys[pick % len(keys)] + judge = key in JUDGE_FIELDS + if key in ("country", "scenario_id", "variable"): + return # a different case key is a dropped plus an added entry + if mutation == "delete": + del entry[key] + else: + entry[key] = [entry[key], "mutated"] + elif mutation == "add_decision_key": + entry["x_extra"] = "added" + elif mutation == "add_judge_key": + if "judged_on_utc" in entry: + del entry["judged_on_utc"] + else: + entry["judged_on_utc"] = "2026-10-08" + else: + decisions = [k for k in keys if k not in JUDGE_FIELDS] + first = decisions[pick % (len(decisions) - 1)] + second = decisions[decisions.index(first) + 1] + items = list(entry.items()) + i, j = keys.index(first), keys.index(second) + items[i], items[j] = items[j], items[i] + staged[index] = entry = dict(items) + rejudged = frozenset({_case(entry)}) if reopened else frozenset() + if reopened and judge: + assert ( + driver.verify_adjudication_changes(base, staged, rejudged, [], ruled_cases) + == 0 + ) + else: + with pytest.raises(SystemExit, match="change recorded decisions"): + driver.verify_adjudication_changes(base, staged, rejudged, [], ruled_cases) + + +# --- Wording amendments ---------------------------------------------------------- + + +def _amendment(entry, old, new, field="reasoning", **extra): + return { + "case_id": _case(entry), + "field": field, + "old": old, + "new": new, + "reason": "The re-judge replaced the verdict this sentence describes.", + **extra, + } + + +def test_a_listed_wording_amendment_is_allowed_and_nothing_else(ruled): + record = ruled.record + entry = _unruled(record)[0] + old = entry["reasoning"].split(". ")[0] + amendment = _amendment(entry, old, old + ", as restated") + original = copy.deepcopy(record) + entry["reasoning"] = entry["reasoning"].replace(old, old + ", as restated") + _write(ruled.staged, record) + assert _verify(ruled, {_case(entry)}, [amendment]) == 0 + # Unlisted, the same change is refused. + with pytest.raises(SystemExit, match="change recorded decisions"): + _verify(ruled, {_case(entry)}) + # Listed but not applied, it is refused too: the list is exact. + _write(ruled.staged, original) + with pytest.raises(SystemExit, match="change recorded decisions"): + _verify(ruled, {_case(entry)}, [amendment]) + # Any other change beside it is still refused. + entry["adjudicated_on"] = "2026-10-06" + _write(ruled.staged, record) + with pytest.raises(SystemExit, match="change recorded decisions"): + _verify(ruled, {_case(entry)}, [amendment]) + + +def test_an_amendment_must_find_its_old_text_exactly_once(ruled): + entry = _unruled(ruled.record)[0] + for old in ("no such wording", " "): + with pytest.raises(SystemExit, match="occurs"): + _verify(ruled, {_case(entry)}, [_amendment(entry, old, "x")]) + + +@pytest.fixture +def amendment_stage(tmp_path): + entry = driver.base_adjudications()[0] + stage = tmp_path / "stage" + stage.mkdir() + (stage / driver.PROMPT_CHANGES).write_text( + json.dumps({"added": [], "changed": [_case(entry)], "kept": []}) + ) + + def write(*items): + (stage / driver.AMENDMENTS).write_text(json.dumps({"amendments": items})) + return driver.load_amendments(stage, frozenset({_case(entry)})) + + return entry, write + + +def test_load_amendments_accepts_wording_of_rejudged_cases(amendment_stage): + entry, write = amendment_stage + items = [ + _amendment(entry, "a", "b"), + _amendment(entry, "a", "b", field="case_annotation"), + _amendment(entry, "a", "b", field="annotation", model=NEW), + ] + assert write(*items) == items + + +@pytest.mark.parametrize( + "defect, message", + [ + ({"field": "adjudicated_failure_source"}, "not wording"), + ({"field": "excluded_from_scoring"}, "not wording"), + ({"field": "judge_failure_source"}, "not wording"), + ({"case_id": "us__scenario_999__snap"}, "not re-judged"), + ({"reason": " "}, "reason"), + ({"new": None}, "new"), + ({"old": ""}, "old"), + ({"new": "same", "old": "same"}, "changes nothing"), + ({"model": NEW}, "keys"), + ({"field": "annotation"}, "keys"), + ], +) +def test_load_amendments_refuses_anything_but_wording_of_rejudged_cases( + amendment_stage, defect, message +): + entry, write = amendment_stage + with pytest.raises(SystemExit, match=message): + write({**_amendment(entry, "a", "b"), **defect}) + + +def test_without_a_stage_list_the_amendments_record_is_release_20261010s(tmp_path): + base = base_blob(ANNOTATIONS_DIR / AMENDMENTS_RECORD) + assert release.committed_amendments(tmp_path) == base + (tmp_path / driver.AMENDMENTS).write_text(json.dumps({"amendments": []})) + assert release.committed_amendments(tmp_path) == base + + +AMENDMENT_ITEMS = st.fixed_dictionaries( + { + "case_id": st.sampled_from(["us__scenario_001__snap", "us__scenario_002__ssi"]), + "field": st.sampled_from(["reasoning", "case_annotation"]), + "old": st.text(min_size=1, max_size=5), + "new": st.text(min_size=1, max_size=5), + "reason": st.text(min_size=1, max_size=5), + } +) + + +@examples(100) +@given( + items=st.lists(AMENDMENT_ITEMS, min_size=1, max_size=4), + note=st.none() | st.text(max_size=8), +) +def test_a_stage_list_is_appended_to_release_20261010s_amendments( + tmp_path_factory, items, note +): + """The earlier amendments stay, in order, before the stage's, and the + note names how many are this release's; the bytes are in the committed + form.""" + stage = tmp_path_factory.mktemp("amend") + listed = ( + {"amendments": items} if note is None else {"note": note, "amendments": items} + ) + (stage / driver.AMENDMENTS).write_text(json.dumps(listed)) + base = json.loads(base_blob(ANNOTATIONS_DIR / AMENDMENTS_RECORD)) + text = release.committed_amendments(stage) + merged = json.loads(text) + assert list(merged) == ["note", "amendments"] + assert merged["amendments"] == base["amendments"] + items + assert merged["note"].startswith( + base["note"] + + f" The last {len(items)} (after the first {len(base['amendments'])})" + ) + if note is not None and note.strip(): + assert merged["note"].endswith(" " + note.strip()) + assert text == json.dumps(merged, indent=1, ensure_ascii=False).encode() + + +def test_the_freeze_commits_the_appended_amendments(frozen_board, monkeypatch): + """A stage list, bound by the receipt, is committed after release + 20261010's amendments; verify_frozen checks the frozen record is that.""" + stage = frozen_board.stage + item = { + "case_id": KEPT, + "field": "case_annotation", + "old": "outdated", + "new": "prior year's", + "reason": "The re-judge reworded it.", + } + (stage / driver.AMENDMENTS).write_text(json.dumps({"amendments": [item]})) + monkeypatch.setattr(driver, "rejudged_cases", lambda stage: frozenset({KEPT})) + receipt = json.loads((stage / "release-ready.json").read_text()) + receipt["files"][driver.AMENDMENTS] = sha(stage / driver.AMENDMENTS) + (stage / "release-ready.json").write_text(json.dumps(receipt)) + # The kept case is now re-opened: its verdict is new, bound to its prompt, + # and the provenance record would list it; both gates are tested above. + monkeypatch.setattr(release, "verify_verdicts", lambda stage: None) + monkeypatch.setattr(release, "verify_judge_provenance", lambda *a: None) + release.main(["--stage-dir", str(stage)]) + frozen = json.loads( + (release.ROOT / ANNOTATIONS_DIR / AMENDMENTS_RECORD).read_text() + ) + base = json.loads(base_blob(ANNOTATIONS_DIR / AMENDMENTS_RECORD)) + assert frozen["amendments"] == [*base["amendments"], item] + assert frozen_board.checked == [[item]] + + +# One re-opened, decided case: its verdict's rationale and diagnoses are the +# sources triage folds into the published case note and row annotations. +WORDING_CASE = "us__scenario_001__snap" +RATIONALE = "The model applied an outdated threshold." +DIAGNOSES = {"m1": "It left out A. More.", "m2": "It used last year's threshold."} +DECISION = { + "country": "us", + "scenario_id": "scenario_001", + "variable": "snap", + "judge_model": "claude-opus-5-5", + "judge_failure_source": "llm_error", + "judge_failure_subtype": "thresholds_rates", + "adjudicated_failure_source": "llm_error", + "adjudicated_failure_subtype": "thresholds_rates", + "adjudicated_on": "2026-10-06", + "adjudicator": "developer", + "reasoning": "The threshold is the current year's.", +} + + +def _row_amendment(old, new, model="m1"): + return { + "case_id": WORDING_CASE, + "field": "annotation", + "model": model, + "old": old, + "new": new, + "reason": "r", + } + + +def _note_amendment(old, new): + return { + "case_id": WORDING_CASE, + "field": "case_annotation", + "old": old, + "new": new, + "reason": "r", + } + + +# A later amendment rewrites words an earlier one wrote. +CHAINED = [ + _row_amendment("It left out A.", "It left out A, as B does."), + _row_amendment("as B does", "as the corrected B does"), + _note_amendment("an outdated", "the prior year's"), +] +# What triage publishes with CHAINED applied, spelled out by hand. +PUBLISHED = { + "m1": "It left out A, as the corrected B does. More.", + "m2": DIAGNOSES["m2"], + "note": "The model applied the prior year's threshold." + + " Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned " + "llm_error; adjudicated llm_error (thresholds_rates). The threshold is the " + "current year's.", +} +# What triage publishes on the case with CHAINED applied when no adjudication +# decides it: the case note is the amended rationale alone. +UNDECIDED = {**PUBLISHED, "note": "The model applied the prior year's threshold."} + + +def _wording_stage(tmp_path, published, decided=True): + """An audit with the case's verdict, the staged record deciding it (or, + with ``decided`` false, deciding nothing), and the published case note + and row annotations ``published`` spells, every column as triage writes + it: the verdict's classes, which the decision affirms, beside the text.""" + import pandas as pd + + from policybench.adjudications import adjudication_sentence + + assert PUBLISHED["note"].endswith(adjudication_sentence(DECISION)) + audit = tmp_path / "audit" + (audit / "cases" / WORDING_CASE).mkdir(parents=True) + keys = {"country": "us", "scenario_id": "scenario_001", "variable": "snap"} + (audit / "cases.jsonl").write_text( + json.dumps( + { + "case_id": WORDING_CASE, + "scenario_id": keys["scenario_id"], + "variable": keys["variable"], + "wrong_models": list(DIAGNOSES), + "parse_failure_only": False, + } + ) + + "\n" + ) + verdict = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": RATIONALE, + "models": [ + { + "model": model, + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": diagnosis, + } + for model, diagnosis in DIAGNOSES.items() + ], + } + (audit / "cases" / WORDING_CASE / "verdict.json").write_text(json.dumps(verdict)) + annotations = tmp_path / "publish" / RUN / "annotations" + annotations.mkdir(parents=True) + (annotations / ADJUDICATIONS).write_text( + json.dumps({"adjudications": [DECISION] if decided else []}) + ) + classes = {"failure_source": "llm_error", "failure_subtype": "thresholds_rates"} + pd.DataFrame( + [ + { + **keys, + "model": model, + **classes, + "reference_suspect": False, + "annotation": published[model], + } + for model in DIAGNOSES + if model in published + ] + ).to_csv(annotations / ROWS, index=False) + note = { + **keys, + "wrong_model_count": len(DIAGNOSES), + "case_failure_sources": "llm_error", + "case_failure_subtypes": "thresholds_rates", + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_annotation": published["note"], + } + pd.DataFrame([note]).to_csv(annotations / NOTES, index=False) + return annotations, audit + + +def test_the_freeze_replays_chained_amendments_onto_the_sources(tmp_path): + """Each text is rebuilt from its verdict source and the adjudication + sentence, the amendments replayed in order; the full result must be the + published text, and a text no amendment names must be its source.""" + annotations, audit = _wording_stage(tmp_path, PUBLISHED) + release.verify_annotation_amendments(annotations, CHAINED, audit) + + +@pytest.mark.parametrize( + "published, amendments", + [ + # The listed fragment is present, beside wording nobody listed. + ({**PUBLISHED, "m1": PUBLISHED["m1"] + " Unlisted."}, CHAINED), + ({**PUBLISHED, "note": "Unlisted. " + PUBLISHED["note"]}, CHAINED), + # A text no amendment names, reworded. + ({**PUBLISHED, "m2": "It used an unlisted threshold."}, CHAINED), + ({**PUBLISHED, "m2": DIAGNOSES["m2"] + " Unlisted."}, CHAINED), + # With no amendments at all, every text is still checked. + ({**PUBLISHED, "m1": DIAGNOSES["m1"] + " Unlisted."}, []), + # A listed amendment not applied, or only half of a chain. + ({**PUBLISHED, "m1": DIAGNOSES["m1"]}, CHAINED), + ({**PUBLISHED, "m1": "It left out A, as B does. More."}, CHAINED), + # A published row annotation dropped. + ({key: text for key, text in PUBLISHED.items() if key != "m2"}, CHAINED), + ], +) +def test_the_freeze_refuses_wording_the_amendments_do_not_produce( + tmp_path, published, amendments +): + annotations, audit = _wording_stage(tmp_path, published) + with pytest.raises(SystemExit, match="is not what triage builds"): + release.verify_annotation_amendments(annotations, amendments, audit) + + +def test_each_replayed_old_text_must_occur_once_when_applied(tmp_path): + """The first amendment makes the second's old text occur twice.""" + twice = [ + _row_amendment("It left out A.", "It left out A. A."), + _row_amendment("A.", "B."), + ] + annotations, audit = _wording_stage( + tmp_path, {**PUBLISHED, "m1": "It left out B. A. More."} + ) + with pytest.raises(SystemExit, match="occurs 2 times"): + release.verify_annotation_amendments(annotations, twice, audit) + + +def test_an_undecided_case_publishes_its_verdict_sources(tmp_path): + annotations, audit = _wording_stage(tmp_path, UNDECIDED, decided=False) + release.verify_annotation_amendments(annotations, CHAINED, audit) + + +# Each column triage publishes beside the wording, changed on the case: its +# rows' classes and flag, and its note's classes, flag, count and hypothesis. +COLUMN_EDITS = [ + (ROWS, "failure_source", "prompt_ambiguity"), + (ROWS, "failure_subtype", "missing_output"), + (ROWS, "reference_suspect", "True"), + (NOTES, "case_failure_sources", "prompt_ambiguity"), + (NOTES, "case_failure_subtypes", "missing_output"), + (NOTES, "reference_suspect", "True"), + (NOTES, "wrong_model_count", "3"), + (NOTES, "reference_bug_hypothesis", "The reference is wrong."), +] + + +@pytest.mark.parametrize("decided", [False, True]) +@pytest.mark.parametrize("name, column, value", COLUMN_EDITS) +def test_the_freeze_refuses_any_column_triage_did_not_write( + tmp_path, decided, name, column, value +): + """The texts unchanged, any other published column that is not the + verdict's (or, on a decided case, the adjudication's) is refused, and + named by its case and column.""" + import pandas as pd + + annotations, audit = _wording_stage( + tmp_path, PUBLISHED if decided else UNDECIDED, decided=decided + ) + frame = pd.read_csv(annotations / name, dtype=str, keep_default_na=False) + frame.loc[0, column] = value + frame.to_csv(annotations / name, index=False) + with pytest.raises( + SystemExit, + match=rf"{re.escape(name)} is not what triage builds.*{WORDING_CASE}\S* " + rf"{column}", + ): + release.verify_annotation_amendments(annotations, CHAINED, audit) + + +@pytest.mark.parametrize("defect", ["dropped_column", "extra_column", "row_order"]) +def test_the_freeze_refuses_columns_or_rows_triage_did_not_write(tmp_path, defect): + import pandas as pd + + annotations, audit = _wording_stage(tmp_path, PUBLISHED) + path = annotations / ROWS + frame = pd.read_csv(path, dtype=str, keep_default_na=False) + if defect == "dropped_column": + frame = frame.drop(columns=["failure_subtype"]) + problem = "its columns" + elif defect == "extra_column": + frame["reviewed"] = "yes" + problem = "its columns" + else: + frame = frame.iloc[::-1] + problem = "its row order or serialization" + frame.to_csv(path, index=False) + with pytest.raises(SystemExit, match=f"differs at.*{problem}"): + release.verify_annotation_amendments(annotations, CHAINED, audit) + + +# --- Treatment ----------------------------------------------------------------- + + +def test_the_registered_treatment_matches_the_run_and_a_drift_is_refused(tmp_path): + state = {"model": NEW, "treatment_fingerprint": NEW_FINGERPRINT} + path = tmp_path / "run_state.json" + scenarios = driver.SNAPSHOT / "scenarios.csv" + release.validate_treatment(freezer, state, path, scenarios) + drifted = copy.deepcopy(state) + drifted["treatment_fingerprint"]["request_timeout_seconds"] = 600 + with pytest.raises(SystemExit, match="disagrees with registry"): + release.validate_treatment(freezer, drifted, path, scenarios) + + +def test_the_live_runs_fingerprint_is_the_one_these_tests_pin(): + """(local: the stage is gitignored)""" + state = REPO / f"results/local/musewonder/stage/inputs/{SLUG}/run_state.json" + if not state.is_file(): + pytest.skip("no live stage in this checkout") + assert json.loads(state.read_text())["treatment_fingerprint"] == NEW_FINGERPRINT From e569ffb6d7950ae64e25cc798f50a12bca165c1a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:22:06 -0400 Subject: [PATCH 09/28] WIP: the Muse Wonder release's design note The base, the rebuilt seed, judge template v2's mixed tree, the provider route, the ten engine-defect candidates with their 2.38.6 targets, the audit's own fix module (r33), DecisionWave and the open decisions. The run and the engine build are pending. Co-Authored-By: Claude Opus 5.5 --- docs/musewonder/design.md | 115 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/musewonder/design.md diff --git a/docs/musewonder/design.md b/docs/musewonder/design.md new file mode 100644 index 00000000..0bd669cf --- /dev/null +++ b/docs/musewonder/design.md @@ -0,0 +1,115 @@ +# Meta Muse Wonder and the third engine upgrade: design + +## What the release is + +This release does two things on top of release `dashboard-data-20261010` (PR #208): + +1. It adds Meta Muse Wonder (display id `muse-wonder`, provider id `openrouter/meta/muse-wonder`, run slug `musewonder`) as the board's 48th model. It is the board's first Meta model. +2. It moves the references from policyengine-us 2.38.6 to the newest policyengine-us release, a third `engine_upgrade` revision (Max, 2026-09-28: "we should be using the latest pe for this always!"). Release 20261010 left ten outputs excluded for engine defects whose upstream fixes had not shipped. Max's instruction of 2026-10-12 for those: "If any have shipped, regenerate their cells instead of excluding them." Each one the new engine fixes is regenerated and scored again. + +It rules no exclusion. Two decisions on reference conventions were open when it was staged: d1252 (VA scenario_039 federal and OH scenario_025 state income tax) and d1263 (the Tax Table against the rate schedule). The spec records both as open, so nothing scored changes on their points. + +Meta also released Muse Wonder Full on 2026-10-12. Only Muse Wonder has an API; Muse Wonder Full is limited to Muse Code subscribers. The board runs every model through its provider's API under one protocol, so Muse Wonder Full cannot join it, and the release note says so. + +The driver is `scripts/finish_musewonder.py`, adapted from `scripts/finish_haiku55.py`. The freeze is `scripts/freeze_musewonder.py`, adapted from `scripts/freeze_haiku55.py`. Their mechanics are release 20261010's (`docs/haiku55/design.md`) except where this note says otherwise. The inputs are in `docs/musewonder/`: + +- `spec.json`: the rulings, no proposals, no adjudications, and the decisions on any outputs the engine upgrade newly excludes (`upgrade_adjudications`, empty unless the build needs one); +- `input_pins.json`: the finished run (written by `--step pin-inputs`); +- `seed_digest.csv`, `seed_recovery.md` and `seed_recovery.json`: the audit seed, rebuilt (below). + +The engine upgrade's builder, actions, evidence and narratives are in `reference_audit/2026-10-12-engine-upgrade/`, adapted from `reference_audit/2026-10-09-engine-upgrade/`. + +| | Release 20261010 | This release before a build is installed | +|---|---|---| +| Models | 47 | 48 | +| Requested outputs per model | 1,984 | 1,984 | +| Exclusions | 58: 42 unlisted inputs, 14 engine defects, 2 later-published law | 58, the same records | +| Scored outputs per model | 1,926 | 1,926 | +| Adjudications | 70, of which 58 exclude | 70, the same decisions | + +An installed build changes these counts as in release 20261010: it removes the records it regenerates and drops their decisions, and adds any records it newly excludes. + +## The base + +The base is pinned in `scripts/finish_musewonder.py`. Each value is the one git holds at `BASE_COMMIT`. + +| Item | Value | +|---|---| +| Base tag, payload sha256 | `dashboard-data-20261010`, `f834478e…a6410` (128,756,688 bytes) | +| Models, outputs, exclusions, scored | 47, 1,984, 58 and 1,926 | +| `reference_outputs.csv` | `fdd17a57…f371` (policyengine-us 2.38.6) | +| `reference_outputs.csv.meta.json` | `d534cfea…c54b` | +| `reference_exclusions.json` | `089e01e1…3626` | +| `scenarios.csv`, its sidecar | `71b16212…858a`, `03a66e90…aebb` | +| `us_case_reference_explanations.csv` | `7ef73f3f…8f5f` | +| `BASE_COMMIT` | `5a8164a0…`, the squash merge of PR #208 on main (2026-10-10) | +| `SEED_RELEASE_COMMIT` | `BASE_COMMIT`: release 20261010's annotations rendered the seed's re-judged prompts | +| Grounding | `b1e4a9bc…b55c`, the main clone's `results/local/unified_audit/grounding.csv` | +| Seed digest | `bc7c057e…b883` (`docs/musewonder/seed_digest.csv`, 677 judged cases) | + +The 58 records were decided on three engines: 38 on policyengine-us 1.755.4, 18 on 2.15.17 and 2 on 2.38.6. + +## The seed + +The seed is release 20261010's judged audit: release 20261006's 674 judged cases with the 232 that release 20261010 re-judged written over them, 677 in all. That release's stage was deleted with its worktree after it shipped, and the repository kept only the hashes of its verdicts and prompts (`docs/haiku55/judge_provenance.json`). `scripts/recover_20261010_audit.py` rebuilt the 232 from each judge's Claude Code session transcript and accepted a transcript only when its prompt and verdict hash to the provenance entry. `docs/musewonder/seed_recovery.md` gives the method and its checks. No verdict text is new. + +The seed tree is `results/local/musewonder/seed-20261010/audit`, with a read-only copy in `~/reviews/policybench-muse-wonder/seed-recovery/seed-20261010`. + +## Judge template v2 + +PR #210 made the judge template versioned. `prepare_audit` keeps a verdict on the template version its sidecar records (none means v1) and renders every new or re-opened case on v2. The seed's 677 verdicts are all v1. This release's tree is therefore mixed: + +- a kept case's prompt is its v1 render, and its verdict is the seed's, byte for byte; +- a re-opened or new case renders on v2 (`JUDGE_TEMPLATE_VERSION = 2`), and `validate_verdicts` requires its sidecar to record version 2. + +`install_references` renders each judged case on the version its sidecar records (`recorded_versions`, `render_audit_with`) to tell an unchanged prompt from a changed one, so a kept v1 case is not re-opened by the template alone. + +## The run and onboarding + +**Provider.** Meta's own Model API serves the model as `muse-wonder-1.0`, but PolicyBench holds no Meta key. OpenRouter lists `meta/muse-wonder` with a single endpoint, whose provider is Meta, serving `meta/muse-wonder-20261012`. So the board calls Meta's model through OpenRouter, as it calls Kimi, GLM, Qwen, MiniMax and Inkling. The card and the config record the route. The price is OpenRouter's list price, $4 per million input tokens and $20 per million output (cache read $0.20). + +**Pending.** The card PR (#216) waits on access: every OpenRouter call to `meta/muse-wonder` returns HTTP 403 until the account confirms the age attestation OpenRouter requires for this model (d1445). Then, in order: + +1. the reasoning probes (plain, forced tool, `tool_choice` auto, required), as `sensitivity/claude-thinking-2026-08.md` did for Claude, recorded on the card; +2. the card's `answer_contract`: a forced answer tool unless the provider rejects it; +3. the onboarding gauntlet and the board run on the frozen 100-household manifest with a budget cap set from the probes' token counts; +4. the `tool_choice` auto sensitivity run, if forcing the tool suppresses reasoning. + +## The engine upgrade + +The mechanics are release 20261010's ("The engine upgrade" in `docs/haiku55/design.md`): the builder reads the base's references from git, recomputes every output on the new engine, and must explain every move by a reviewed action. `sweep_timing.py` requires the reference engine to be the newest release when the sweep begins, and re-sweeps any newer release before publication. + +**The ten candidates.** Release 20261010 kept ten engine-defect records excluded because their fixes had not shipped. Their audited targets on 2.38.6 are in `reference_audit/2026-10-12-engine-upgrade/evidence/pe2.38.6.json` (engine value, then corrected value): + +| Output | Root cause | Fix | 2.38.6 | Target | Target kind | +|---|---|---|---|---|---| +| scenario_007 (ID) state income tax | r07 Idaho health premiums | #10023 | 771.56 | 661.32 | fix_modules | +| scenario_042 (WI) state income tax | r06, r32 | #10026 | 464.18 | 0.00 | record | +| scenario_053 (ID) state income tax | r07 | #10023 | 2,451.06 | 2,186.06 | fix_modules | +| scenario_080 SNAP | r30 heat-and-eat SUA | #10033 | 3,576.00 | 3,240.00 | record | +| scenario_091 (WI) state income tax | r32 | #10026 | 895.81 | 878.52 | record | +| scenario_099 (CA) state income tax | r02, r11, r33 | #10032, #10034, #10052 | 4,588.48 | 4,615.48 | fix_modules | +| scenario_100 federal refundable credits | r08 EITC deferrals | #10030 | 2,878.10 | 822.29 | record | +| scenario_100 state refundable credits | r08 | #10030 | 473.18 | 164.46 | record | +| scenario_119 federal refundable credits | r08 | #10030 | 2,854.01 | 3,366.07 | record | +| scenario_119 state income tax | r08 | #10030 | 1,859.09 | 1,756.68 | record | + +A `record` target is the exclusion record's alternative value. A `fix_modules` target is what the audited fix modules give on 2.38.6, where the defect is still present, because something else in the engine moved the output since the record was decided. + +**This audit's own fix module.** CA scenario_099's record names two root causes, r02 and r11. Release 20261010's recheck on 2.38.6 found a third difference: California does not conform to the federal educator-expense deduction, so it adds the deduction back (`r33_ca_educator_expense_addback.py` in `reference_audit/2026-10-12-engine-upgrade/fixes/`, mirroring PolicyEngine/policyengine-us#10052's `ca_educator_expense_addition`). Earlier upgrades read every module from `reference_audit/2026-09-22/fixes` at `BASE_COMMIT`. This one may also pin a module from its own `fixes/`, read at HEAD (`fix_location` in the builder, `fix_module_blob` in the driver), and the pin then carries its `path`. A name found under both folders resolves to the 2026-09-22 module, so a local module cannot shadow an audited one. `freeze_snapshot.verify_regeneration_target_pins` requires each pinned path to be a `reference_audit//fixes/` file at its committed bytes. + +**First pass on a pre-release.** On policyengine-us 2.41.0 built from git (`de3815f4cc`, before its PyPI publish), 23 outputs differ from 2.38.6, all of them excluded. All ten candidates land on their targets, and no scored output moves. The release build waits on the published wheel, which `sweep_timing.py` requires. + +**DecisionWave.** Release 20261010's driver dated every triage decision by its ruled 2026-10-06 wave. This release rules nothing, so a triage decision (a developer affirming a reference the upgrade changed, which a re-judged verdict flagged) is dated by the engine upgrade's revision day. The upgrade's day is named as a wave in the date conventions whenever the upgrade decides anything, a new exclusion or a triage affirmation (`decision_wave`, `release_date_conventions`), with the day its decisions were written (`upgrade_adjudications_written_on`). With no decision, the conventions stay release 20261010's. + +**Builder notes.** A sidecar keeps every upgrade's revision. `scripts/freeze_snapshot.py` accepts both later builders' notes (`UPGRADE_BUILDERS`): the 2026-10-10 upgrade's and this one's. Both pin the same inherited setup (the 2026-09-29 conventions, composer and adapter, `latest_final.py` and the sales-tax table). + +## Open decisions checked before the freeze + +| Decision | State when staged | Effect here | +|---|---|---| +| d1252 (keep scoring VA scenario_039 federal and OH scenario_025 state income tax) | open | none: both stay scored as in release 20261010 | +| d1263 (rate schedule against the Tax Table) | open | none: federal references keep the section 1 rate schedule | +| d1441 (add Muse Spark 1.3 in this release) | open | not added without Max's yes | + +Check `cos decisions --all` again before freezing and apply any ruling. From 85a359312ef91ce30c7a73801120486c7eaa5adf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:22:06 -0400 Subject: [PATCH 10/28] Add Meta as a provider and name Muse Wonder in the app and the paper The board's first Meta model: its label, order and frontier group, the provider's label and mark (@lobehub/icons Meta.Color, in 5.10.0), and its paper display name. Co-Authored-By: Claude Opus 5.5 --- app/src/components/ProviderMark.tsx | 2 ++ app/src/modelMeta.ts | 6 ++++++ app/tests/modelMeta.test.ts | 2 ++ policybench/paper_results.py | 1 + 4 files changed, 11 insertions(+) diff --git a/app/src/components/ProviderMark.tsx b/app/src/components/ProviderMark.tsx index 1bf65ea8..d5678593 100644 --- a/app/src/components/ProviderMark.tsx +++ b/app/src/components/ProviderMark.tsx @@ -3,6 +3,7 @@ import { Anthropic, DeepSeek, Google, + Meta, Minimax, Moonshot, OpenAI, @@ -78,6 +79,7 @@ export default function ProviderMark({ {provider === "anthropic" &&